appwrite-utils-cli 1.11.0 → 1.12.0

package/dist/migrations/dataLoader.js

@@ -0,0 +1,1272 @@
+import { AttributeMappingsSchema, CollectionCreateSchema, importDefSchema, } from "appwrite-utils";
+import path from "path";
+import fs from "fs";
+import { convertObjectByAttributeMappings } from "appwrite-utils-helpers";
+import { z } from "zod";
+import { checkForCollection } from "../collections/methods.js";
+import { ID, Users } from "node-appwrite";
+import { logger, AdapterFactory, MessageFormatter } from "appwrite-utils-helpers";
+import { findOrCreateOperation, updateOperation } from "../shared/migrationHelpers.js";
+import { AuthUserCreateSchema } from "../schemas/authUser.js";
+import { UsersController } from "../users/methods.js";
+import { isEmpty } from "es-toolkit/compat";
+// Define a schema for the structure of collection import data using Zod for validation
+export const CollectionImportDataSchema = z.object({
+    // Optional collection creation schema
+    collection: CollectionCreateSchema.optional(),
+    // Array of data objects each containing rawData, finalData, context, and an import definition
+    data: z.array(z.object({
+        rawData: z.any(), // The initial raw data
+        finalData: z.any(), // The transformed data ready for import
+        context: z.any(), // Additional context for the data transformation
+        importDef: importDefSchema.optional(), // The import definition schema
+    })),
+});
+// DataLoader class to handle the loading of data into collections
+export class DataLoader {
+    // Private member variables to hold configuration and state
+    appwriteFolderPath;
+    importDataActions;
+    database;
+    usersController;
+    config;
+    // Map to hold the import data for each collection by name
+    importMap = new Map();
+    // Map to track old to new ID mappings for each collection, if applicable
+    oldIdToNewIdPerCollectionMap = new Map();
+    // Map to hold the import operation ID for each collection
+    collectionImportOperations = new Map();
+    // Map to hold the merged user map for relationship resolution
+    // Will hold an array of the old user ID's that are mapped to the same new user ID
+    // For example, if there are two users with the same email, they will both be mapped to the same new user ID
+    // Prevents duplicate users with the other two maps below it and allows me to keep the old ID's
+    mergedUserMap = new Map();
+    // Maps to hold email and phone to user ID mappings for unique-ness in User Accounts
+    emailToUserIdMap = new Map();
+    phoneToUserIdMap = new Map();
+    userIdSet = new Set();
+    userExistsMap = new Map();
+    shouldWriteFile = false;
+    _adapter = null;
+    async getAdapter() {
+        if (!this._adapter) {
+            const { adapter } = await AdapterFactory.createFromConfig(this.config);
+            this._adapter = adapter;
+        }
+        return this._adapter;
+    }
+    // Constructor to initialize the DataLoader with necessary configurations
+    constructor(appwriteFolderPath, importDataActions, database, config, shouldWriteFile) {
+        this.appwriteFolderPath = appwriteFolderPath;
+        this.importDataActions = importDataActions;
+        this.database = database;
+        this.usersController = new UsersController(config, database);
+        this.config = config;
+        this.shouldWriteFile = shouldWriteFile || false;
+    }
+    // Helper method to generate a consistent key for collections
+    getCollectionKey(name) {
+        return name.toLowerCase().replace(" ", "");
+    }
+    /**
+     * Merges two objects by updating the source object with the target object's values.
+     * It iterates through the target object's keys and updates the source object if:
+     * - The source object has the key.
+     * - The target object's value for that key is not null, undefined, or an empty string.
+     * - If the target object has an array value, it concatenates the values and removes duplicates.
+     *
+     * @param source - The source object to be updated.
+     * @param target - The target object with values to update the source object.
+     * @returns The updated source object.
+     */
+    mergeObjects(source, update) {
+        // Create a new object to hold the merged result
+        const result = { ...source };
+        // Loop through the keys of the object we care about
+        for (const [key, value] of Object.entries(source)) {
+            // Check if the key exists in the target object
+            if (!Object.hasOwn(update, key)) {
+                // If the key doesn't exist, we can just skip it like bad cheese
+                continue;
+            }
+            if (update[key] === value) {
+                continue;
+            }
+            // If the value ain't here, we can just do whatever man
+            if (value === undefined || value === null || value === "") {
+                // If the update key is defined
+                if (update[key] !== undefined &&
+                    update[key] !== null &&
+                    update[key] !== "") {
+                    // might as well use it eh?
+                    result[key] = update[key];
+                }
+                // ELSE if the value is an array, because it would then not be === to those things above
+            }
+            else if (Array.isArray(value)) {
+                // Get the update value
+                const updateValue = update[key];
+                // If the update value is an array, concatenate and remove duplicates
+                // and poopy data
+                if (Array.isArray(updateValue)) {
+                    result[key] = [...new Set([...value, ...updateValue])].filter((item) => item !== null && item !== undefined && item !== "");
+                }
+                else {
+                    // If the update value is not an array, just use it
+                    result[key] = [...value, updateValue].filter((item) => item !== null && item !== undefined && item !== "");
+                }
+            }
+            else if (typeof value === "object" && !Array.isArray(value)) {
+                // If the value is an object, we need to merge it
+                if (typeof update[key] === "object" && !Array.isArray(update[key])) {
+                    result[key] = this.mergeObjects(value, update[key]);
+                }
+            }
+            else {
+                // Finally, the source value is defined, and not an array, so we don't care about the update value
+                continue;
+            }
+        }
+        // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
+        for (const [key, value] of Object.entries(update)) {
+            if (value === undefined || value === null || value === "") {
+                continue;
+            }
+            else if (!Object.hasOwn(source, key)) {
+                result[key] = value;
+            }
+            else if (typeof source[key] === "object" &&
+                typeof value === "object" &&
+                !Array.isArray(source[key]) &&
+                !Array.isArray(value)) {
+                result[key] = this.mergeObjects(source[key], value);
+            }
+            else if (Array.isArray(source[key]) && Array.isArray(value)) {
+                result[key] = [...new Set([...source[key], ...value])].filter((item) => item !== null && item !== undefined && item !== "");
+            }
+            else if (source[key] === undefined ||
+                source[key] === null ||
+                source[key] === "") {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    // Method to load data from a file specified in the import definition
+    loadData(importDef) {
+        // Simply join appwriteFolderPath with the importDef.filePath
+        const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
+        MessageFormatter.info(`Loading data from: ${filePath}`, { prefix: "Data" });
+        if (!fs.existsSync(filePath)) {
+            MessageFormatter.error(`File not found: ${filePath}`, undefined, { prefix: "Data" });
+            return [];
+        }
+        // Read the file and parse the JSON data
+        const rawData = fs.readFileSync(filePath, "utf8");
+        const parsedData = importDef.basePath
+            ? JSON.parse(rawData)[importDef.basePath]
+            : JSON.parse(rawData);
+        MessageFormatter.success(`Loaded ${parsedData?.length || 0} items from ${filePath}`, { prefix: "Data" });
+        return parsedData;
+    }
+    // Helper method to check if a new ID already exists in the old-to-new ID map
+    checkMapValuesForId(newId, collectionName) {
+        const oldIdMap = this.oldIdToNewIdPerCollectionMap.get(collectionName);
+        for (const [key, value] of oldIdMap?.entries() || []) {
+            if (value === newId) {
+                return key;
+            }
+        }
+        return false;
+    }
+    // Method to generate a unique ID that doesn't conflict with existing IDs
+    getTrueUniqueId(collectionName) {
+        let newId = ID.unique();
+        let condition = this.checkMapValuesForId(newId, collectionName) ||
+            this.userExistsMap.has(newId) ||
+            this.userIdSet.has(newId) ||
+            this.importMap
+                .get(this.getCollectionKey("users"))
+                ?.data.some((user) => user.finalData.docId === newId || user.finalData.userId === newId);
+        while (condition) {
+            newId = ID.unique();
+            condition =
+                this.checkMapValuesForId(newId, collectionName) ||
+                    this.userExistsMap.has(newId) ||
+                    this.userIdSet.has(newId) ||
+                    this.importMap
+                        .get(this.getCollectionKey("users"))
+                        ?.data.some((user) => user.finalData.docId === newId || user.finalData.userId === newId);
+        }
+        return newId;
+    }
+    // Method to create a context object for data transformation
+    createContext(db, collection, item, docId) {
+        return {
+            ...item, // Spread the item data for easy access to its properties
+            dbId: db.$id,
+            dbName: db.name,
+            collId: collection.$id,
+            collName: collection.name,
+            docId: docId,
+            createdDoc: {}, // Initially null, to be updated when the document is created
+        };
+    }
+    /**
+     * Transforms the given item based on the provided attribute mappings.
+     * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
+     *
+     * @param item - The item to be transformed.
+     * @param attributeMappings - The mappings that define how each attribute should be transformed.
+     * @returns The transformed item.
+     */
+    transformData(item, attributeMappings) {
+        // Convert the item using the attribute mappings provided
+        const convertedItem = convertObjectByAttributeMappings(item, attributeMappings);
+        // Run additional converter functions on the converted item, if any
+        return this.importDataActions.runConverterFunctions(convertedItem, attributeMappings);
+    }
+    async setupMaps(dbId, specificCollections) {
+        // Initialize the users collection in the import map
+        this.importMap.set(this.getCollectionKey("users"), {
+            data: [],
+        });
+        for (const db of this.config.databases) {
+            if (db.$id !== dbId) {
+                continue;
+            }
+            if (!this.config.collections) {
+                continue;
+            }
+            for (let index = 0; index < this.config.collections.length; index++) {
+                const collectionConfig = this.config.collections[index];
+                let collection = CollectionCreateSchema.parse(collectionConfig);
+                // Skip collections not in the specific list if one was provided
+                if (specificCollections && specificCollections.length > 0 &&
+                    !specificCollections.includes(collection.name)) {
+                    continue;
+                }
+                // Check if the collection exists in the database
+                const collectionExists = await checkForCollection(this.database, db.$id, collection);
+                if (!collectionExists) {
+                    logger.error(`No collection found for ${collection.name}`);
+                    continue;
+                }
+                else if (!collection.name) {
+                    logger.error(`Collection ${collection.name} has no name`);
+                    continue;
+                }
+                // Update the collection ID with the existing one
+                collectionConfig.$id = collectionExists.$id;
+                collection.$id = collectionExists.$id;
+                this.config.collections[index] = collectionConfig;
+                // Find or create an import operation for the collection (non-fatal)
+                try {
+                    const adapter = await this.getAdapter();
+                    const collectionImportOperation = await findOrCreateOperation(adapter, dbId, "importData", collection.$id);
+                    this.collectionImportOperations.set(this.getCollectionKey(collection.name), collectionImportOperation.$id);
+                }
+                catch (error) {
+                    MessageFormatter.warning(`Operations tracking unavailable for ${collection.name}, import will proceed without it`, { prefix: "Import" });
+                }
+                // Initialize the collection in the import map
+                this.importMap.set(this.getCollectionKey(collection.name), {
+                    collection: collection,
+                    data: [],
+                });
+            }
+        }
+    }
+    async getAllUsers() {
+        const users = new UsersController(this.config, this.database);
+        const allUsers = await users.getAllUsers();
+        // Iterate over the users and setup our maps ahead of time for email and phone
+        for (const user of allUsers) {
+            if (user.email) {
+                this.emailToUserIdMap.set(user.email.toLowerCase(), user.$id);
+            }
+            if (user.phone) {
+                this.phoneToUserIdMap.set(user.phone, user.$id);
+            }
+            this.userExistsMap.set(user.$id, true);
+            this.userIdSet.add(user.$id);
+            let importData = this.importMap.get(this.getCollectionKey("users"));
+            if (!importData) {
+                importData = {
+                    data: [],
+                };
+            }
+            importData.data.push({
+                finalData: {
+                    ...user,
+                    email: user.email?.toLowerCase(),
+                    userId: user.$id,
+                    docId: user.$id,
+                },
+                context: {
+                    ...user,
+                    email: user.email?.toLowerCase(),
+                    userId: user.$id,
+                    docId: user.$id,
+                },
+                rawData: user,
+            });
+            this.importMap.set(this.getCollectionKey("users"), importData);
+        }
+        return allUsers;
+    }
+    // Main method to start the data loading process for a given database ID
+    async start(dbId, specificCollections) {
+        MessageFormatter.divider();
+        MessageFormatter.info(`Starting data setup for database: ${dbId}`, { prefix: "Data" });
+        MessageFormatter.divider();
+        // Only fetch users if we're importing users or no specific collections specified
+        const needsUsers = !specificCollections || specificCollections.length === 0 ||
+            specificCollections.some(c => this.getCollectionKey(c) === this.getCollectionKey(this.config.usersCollectionName));
+        if (needsUsers) {
+            const allUsers = await this.getAllUsers();
+            MessageFormatter.info(`Fetched ${allUsers.length} users`, { prefix: "Data" });
+        }
+        // Iterate over the configured databases to find the matching one
+        for (const db of this.config.databases) {
+            if (db.$id !== dbId) {
+                continue;
+            }
+            if (!this.config.collections) {
+                continue;
+            }
+            // Iterate over the configured collections to process each
+            for (const collectionConfig of this.config.collections) {
+                const collection = collectionConfig;
+                // Skip collections not in the specific list
+                if (specificCollections && specificCollections.length > 0 &&
+                    !specificCollections.includes(collection.name)) {
+                    continue;
+                }
+                // Determine if this is the users collection
+                let isUsersCollection = this.getCollectionKey(this.config.usersCollectionName) ===
+                    this.getCollectionKey(collection.name);
+                const collectionDefs = collection.importDefs;
+                if (!collectionDefs || !collectionDefs.length) {
+                    continue;
+                }
+                // Process create and update definitions for the collection
+                const createDefs = collection.importDefs.filter((def) => def.type === "create" || !def.type);
+                const updateDefs = collection.importDefs.filter((def) => def.type === "update");
+                for (const createDef of createDefs) {
+                    if (!isUsersCollection || !createDef.createUsers) {
+                        await this.prepareCreateData(db, collection, createDef);
+                    }
+                    else {
+                        // Special handling for users collection if needed
+                        await this.prepareUserCollectionCreateData(db, collection, createDef);
+                    }
+                }
+                for (const updateDef of updateDefs) {
+                    if (!this.importMap.has(this.getCollectionKey(collection.name))) {
+                        logger.error(`No data found for collection ${collection.name} for updateDef but it says it's supposed to have one...`);
+                        continue;
+                    }
+                    // Prepare the update data for the collection
+                    this.prepareUpdateData(db, collection, updateDef);
+                }
+            }
+            MessageFormatter.info("Running update references", { prefix: "Data" });
+            // this.dealWithMergedUsers();
+            this.updateOldReferencesForNew();
+            MessageFormatter.success("Done running update references", { prefix: "Data" });
+        }
+        // for (const collection of this.config.collections) {
+        //   this.resolveDataItemRelationships(collection);
+        // }
+        MessageFormatter.divider();
+        MessageFormatter.success(`Data setup for database: ${dbId} completed`, { prefix: "Data" });
+        MessageFormatter.divider();
+        if (this.shouldWriteFile) {
+            this.writeMapsToJsonFile();
+        }
+    }
+    /**
+     * Deals with merged users by iterating through all collections in the configuration.
+     * We have merged users if there are duplicate emails or phones in the import data.
+     * This function will iterate through all collections that are the same name as the
+     * users collection and pull out their primaryKeyField's. It will then loop through
+     * each collection and find any documents that have a
+     *
+     * @return {void} This function does not return anything.
+     */
+    // dealWithMergedUsers() {
+    //   const usersCollectionKey = this.getCollectionKey(
+    //     this.config.usersCollectionName
+    //   );
+    //   const usersCollectionData = this.importMap.get(usersCollectionKey);
+    //   if (!this.config.collections) {
+    //     console.log("No collections found in configuration.");
+    //     return;
+    //   }
+    //   let needsUpdate = false;
+    //   let numUpdates = 0;
+    //   for (const collectionConfig of this.config.collections) {
+    //     const collectionKey = this.getCollectionKey(collectionConfig.name);
+    //     const collectionData = this.importMap.get(collectionKey);
+    //     const collectionImportDefs = collectionConfig.importDefs;
+    //     const collectionIdMappings = collectionImportDefs
+    //       .map((importDef) => importDef.idMappings)
+    //       .flat()
+    //       .filter((idMapping) => idMapping !== undefined && idMapping !== null);
+    //     if (!collectionData || !collectionData.data) continue;
+    //     for (const dataItem of collectionData.data) {
+    //       for (const idMapping of collectionIdMappings) {
+    //         // We know it's the users collection here
+    //         if (this.getCollectionKey(idMapping.targetCollection) === usersCollectionKey) {
+    //           const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
+    //           if (targetFieldKey === )
+    //           const targetValue = dataItem.finalData[targetFieldKey];
+    //           const targetCollectionData = this.importMap.get(this.getCollectionKey(idMapping.targetCollection));
+    //           if (!targetCollectionData || !targetCollectionData.data) continue;
+    //           const foundData = targetCollectionData.data.filter(({ context }) => {
+    //             const targetValue = context[targetFieldKey];
+    //             const isMatch = `${targetValue}` === `${valueToMatch}`;
+    //             return isMatch && targetValue !== undefined && targetValue !== null;
+    //           });
+    //         }
+    //       }
+    //     }
+    //   }
+    // }
+    /**
+     * Gets the value to match for a given key in the final data or context.
+     * @param finalData - The final data object.
+     * @param context - The context object.
+     * @param key - The key to get the value for.
+     * @returns The value to match for from finalData or Context
+     */
+    getValueFromData(finalData, context, key) {
+        if (context[key] !== undefined &&
+            context[key] !== null &&
+            context[key] !== "") {
+            return context[key];
+        }
+        return finalData[key];
+    }
+    updateOldReferencesForNew() {
+        if (!this.config.collections) {
+            return;
+        }
+        for (const collectionConfig of this.config.collections) {
+            const collectionKey = this.getCollectionKey(collectionConfig.name);
+            const collectionData = this.importMap.get(collectionKey);
+            if (!collectionData || !collectionData.data)
+                continue;
+            MessageFormatter.processing(`Updating references for collection: ${collectionConfig.name}`, { prefix: "Data" });
+            let needsUpdate = false;
+            // Iterate over each data item in the current collection
+            for (let i = 0; i < collectionData.data.length; i++) {
+                if (collectionConfig.importDefs) {
+                    for (const importDef of collectionConfig.importDefs) {
+                        if (importDef.idMappings) {
+                            for (const idMapping of importDef.idMappings) {
+                                const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
+                                const fieldToSetKey = idMapping.fieldToSet || idMapping.sourceField;
+                                const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
+                                const sourceValue = this.getValueFromData(collectionData.data[i].finalData, collectionData.data[i].context, idMapping.sourceField);
+                                // Skip if value to match is missing or empty
+                                if (!sourceValue ||
+                                    isEmpty(sourceValue) ||
+                                    sourceValue === null)
+                                    continue;
+                                const isFieldToSetArray = collectionConfig.attributes.find((attribute) => attribute.key === fieldToSetKey)?.array;
+                                const targetCollectionData = this.importMap.get(targetCollectionKey);
+                                if (!targetCollectionData || !targetCollectionData.data)
+                                    continue;
+                                // Handle cases where sourceValue is an array
+                                const sourceValues = Array.isArray(sourceValue)
+                                    ? sourceValue.map((sourceValue) => `${sourceValue}`)
+                                    : [`${sourceValue}`];
+                                let newData = [];
+                                for (const valueToMatch of sourceValues) {
+                                    // Find matching data in the target collection
+                                    const foundData = targetCollectionData.data.filter(({ context, finalData }) => {
+                                        const targetValue = this.getValueFromData(finalData, context, targetFieldKey);
+                                        const isMatch = `${targetValue}` === `${valueToMatch}`;
+                                        // Ensure the targetValue is defined and not null
+                                        return (isMatch &&
+                                            targetValue !== undefined &&
+                                            targetValue !== null);
+                                    });
+                                    if (foundData.length) {
+                                        newData.push(...foundData.map((data) => {
+                                            const newValue = this.getValueFromData(data.finalData, data.context, idMapping.targetField);
+                                            return newValue;
+                                        }));
+                                    }
+                                    else {
+                                        logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
+                                    }
+                                    continue;
+                                }
+                                const getCurrentDataFiltered = (currentData) => {
+                                    if (Array.isArray(currentData.finalData[fieldToSetKey])) {
+                                        return currentData.finalData[fieldToSetKey].filter((data) => !sourceValues.includes(`${data}`));
+                                    }
+                                    return currentData.finalData[fieldToSetKey];
+                                };
+                                // Get the current data to be updated
+                                const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
+                                if (newData.length) {
+                                    needsUpdate = true;
+                                    // Handle cases where current data is an array
+                                    if (isFieldToSetArray) {
+                                        if (!currentDataFiltered) {
+                                            // Set new data if current data is undefined
+                                            collectionData.data[i].finalData[fieldToSetKey] =
+                                                Array.isArray(newData) ? newData : [newData];
+                                        }
+                                        else {
+                                            if (Array.isArray(currentDataFiltered)) {
+                                                // Convert current data to array and merge if new data is non-empty array
+                                                collectionData.data[i].finalData[fieldToSetKey] = [
+                                                    ...new Set([...currentDataFiltered, ...newData].filter((value) => value !== null &&
+                                                        value !== undefined &&
+                                                        value !== "")),
+                                                ];
+                                            }
+                                            else {
+                                                // Merge arrays if new data is non-empty array and filter for uniqueness
+                                                collectionData.data[i].finalData[fieldToSetKey] = [
+                                                    ...new Set([
+                                                        ...(Array.isArray(currentDataFiltered)
+                                                            ? currentDataFiltered
+                                                            : [currentDataFiltered]),
+                                                        ...newData,
+                                                    ].filter((value) => value !== null &&
+                                                        value !== undefined &&
+                                                        value !== "" &&
+                                                        !sourceValues.includes(`${value}`))),
+                                                ];
+                                            }
+                                        }
+                                    }
+                                    else {
+                                        if (!currentDataFiltered) {
+                                            // Set new data if current data is undefined
+                                            collectionData.data[i].finalData[fieldToSetKey] =
+                                                Array.isArray(newData) ? newData[0] : newData;
+                                        }
+                                        else if (Array.isArray(newData) && newData.length > 0) {
+                                            // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
+                                            // and take the first value, because it's an array and the attribute is not an array
+                                            collectionData.data[i].finalData[fieldToSetKey] = [
+                                                ...new Set([currentDataFiltered, ...newData].filter((value) => value !== null &&
+                                                    value !== undefined &&
+                                                    value !== "" &&
+                                                    !sourceValues.includes(`${value}`))),
+                                            ].slice(0, 1)[0];
+                                        }
+                                        else if (!Array.isArray(newData) &&
+                                            newData !== undefined) {
+                                            // Simply update the field if new data is not an array and defined
+                                            collectionData.data[i].finalData[fieldToSetKey] = newData;
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            // Update the import map if any changes were made
+            if (needsUpdate) {
+                this.importMap.set(collectionKey, collectionData);
+            }
+        }
+    }
+    writeMapsToJsonFile() {
+        const outputDir = path.resolve(process.cwd(), "zlogs");
+        // Ensure the logs directory exists
+        if (!fs.existsSync(outputDir)) {
+            fs.mkdirSync(outputDir);
+        }
+        // Helper function to write data to a file
+        const writeToFile = (fileName, data) => {
+            const outputFile = path.join(outputDir, fileName);
+            fs.writeFile(outputFile, JSON.stringify(data, null, 2), "utf8", (err) => {
+                if (err) {
+                    MessageFormatter.error(`Error writing data to ${fileName}`, err instanceof Error ? err : new Error(String(err)), { prefix: "Data" });
+                    return;
+                }
+                MessageFormatter.success(`Data successfully written to ${fileName}`, { prefix: "Data" });
+            });
+        };
+        // Convert Maps to arrays of entries for serialization
+        const oldIdToNewIdPerCollectionMap = Array.from(this.oldIdToNewIdPerCollectionMap.entries()).map(([key, value]) => {
+            return {
+                collection: key,
+                data: Array.from(value.entries()),
+            };
+        });
+        const mergedUserMap = Array.from(this.mergedUserMap.entries());
+        // Write each part to a separate file
+        writeToFile("oldIdToNewIdPerCollectionMap.json", oldIdToNewIdPerCollectionMap);
+        writeToFile("mergedUserMap.json", mergedUserMap);
+        // Write each collection's data to a separate file
+        this.importMap.forEach((value, key) => {
+            const data = {
+                collection: key,
+                data: value.data.map((item) => {
+                    return {
+                        finalData: item.finalData,
+                        context: item.context,
+                    };
+                }),
+            };
+            writeToFile(`${key}.json`, data);
+        });
+    }
+    /**
+     * Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,
+     * and then returning the transformed item without user-specific keys.
+     *
+     * @param item - The raw item to be processed.
+     * @param attributeMappings - The attribute mappings for the item.
+     * @returns The transformed item with user-specific keys removed.
+     */
+    prepareUserData(item, attributeMappings, primaryKeyField, newId) {
+        if (this.userIdSet.has(newId) ||
+            this.userExistsMap.has(newId) ||
+            Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+            Array.from(this.phoneToUserIdMap.values()).includes(newId)) {
+            newId = this.getTrueUniqueId(this.getCollectionKey("users"));
+        }
+        let transformedItem = this.transformData(item, attributeMappings);
+        let userData = AuthUserCreateSchema.safeParse(transformedItem);
+        if (userData.data?.email) {
+            userData.data.email = userData.data.email.toLowerCase();
+        }
+        if (!userData.success || !(userData.data.email || userData.data.phone)) {
+            logger.error(`Invalid user data: ${JSON.stringify(userData.error?.issues, undefined, 2)} or missing email/phone`);
+            const userKeys = ["email", "phone", "name", "labels", "prefs"];
+            userKeys.forEach((key) => {
+                if (transformedItem.hasOwnProperty(key)) {
+                    delete transformedItem[key];
+                }
+            });
+            return {
+                transformedItem,
+                existingId: undefined,
+                userData: {
+                    rawData: item,
+                    finalData: transformedItem,
+                },
+            };
+        }
+        const email = userData.data.email?.toLowerCase();
+        const phone = userData.data.phone;
+        let existingId;
+        // Check for duplicate email and phone
+        if (email && this.emailToUserIdMap.has(email)) {
+            existingId = this.emailToUserIdMap.get(email);
+            if (phone && !this.phoneToUserIdMap.has(phone)) {
+                this.phoneToUserIdMap.set(phone, newId);
+            }
+        }
+        else if (phone && this.phoneToUserIdMap.has(phone)) {
+            existingId = this.phoneToUserIdMap.get(phone);
+            if (email && !this.emailToUserIdMap.has(email)) {
+                this.emailToUserIdMap.set(email, newId);
+            }
+        }
+        else {
+            if (email)
+                this.emailToUserIdMap.set(email, newId);
+            if (phone)
+                this.phoneToUserIdMap.set(phone, newId);
+        }
+        if (existingId) {
+            userData.data.userId = existingId;
+            const mergedUsers = this.mergedUserMap.get(existingId) || [];
+            mergedUsers.push(`${item[primaryKeyField]}`);
+            this.mergedUserMap.set(existingId, mergedUsers);
+            const userFound = this.importMap
+                .get(this.getCollectionKey("users"))
+                ?.data.find((userDataExisting) => {
+                let userIdToMatch;
+                if (userDataExisting?.finalData?.userId) {
+                    userIdToMatch = userDataExisting?.finalData?.userId;
+                }
+                else if (userDataExisting?.finalData?.docId) {
+                    userIdToMatch = userDataExisting?.finalData?.docId;
+                }
+                else if (userDataExisting?.context?.userId) {
+                    userIdToMatch = userDataExisting.context.userId;
+                }
+                else if (userDataExisting?.context?.docId) {
+                    userIdToMatch = userDataExisting.context.docId;
+                }
+                return userIdToMatch === existingId;
+            });
+            if (userFound) {
+                userFound.finalData.userId = existingId;
+                userFound.finalData.docId = existingId;
+                this.userIdSet.add(existingId);
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
+            }
+            const userKeys = ["email", "phone", "name", "labels", "prefs"];
+            userKeys.forEach((key) => {
+                if (transformedItem.hasOwnProperty(key)) {
+                    delete transformedItem[key];
+                }
+            });
+            return {
+                transformedItem,
+                existingId,
+                userData: {
+                    rawData: userFound?.rawData,
+                    finalData: userFound?.finalData,
+                },
+            };
+        }
+        else {
+            existingId = newId;
+            userData.data.userId = existingId;
+        }
+        const userKeys = ["email", "phone", "name", "labels", "prefs"];
+        userKeys.forEach((key) => {
+            if (transformedItem.hasOwnProperty(key)) {
+                delete transformedItem[key];
+            }
+        });
+        const usersMap = this.importMap.get(this.getCollectionKey("users"));
+        const userDataToAdd = {
+            rawData: item,
+            finalData: userData.data,
+            context: {},
+        };
+        this.importMap.set(this.getCollectionKey("users"), {
+            data: [...(usersMap?.data || []), userDataToAdd],
+        });
+        this.userIdSet.add(existingId);
+        return {
+            transformedItem,
+            existingId,
+            userData: userDataToAdd,
+        };
+    }
+    /**
+     * Prepares the data for creating user collection documents.
+     * This involves loading the data, transforming it according to the import definition,
+     * and handling the creation of new unique IDs for each item.
+     *
+     * @param db - The database configuration.
+     * @param collection - The collection configuration.
+     * @param importDef - The import definition containing the attribute mappings and other relevant info.
+     */
+    async prepareUserCollectionCreateData(db, collection, importDef) {
+        // Load the raw data based on the import definition
+        const rawData = this.loadData(importDef);
+        let operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
+        // Initialize a new map for old ID to new ID mappings
+        const oldIdToNewIdMap = new Map();
+        // Retrieve or initialize the collection-specific old ID to new ID map
+        const collectionOldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(this.getCollectionKey(collection.name)) ||
+            this.oldIdToNewIdPerCollectionMap
+                .set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
+                .get(this.getCollectionKey(collection.name));
+        const adapter = await this.getAdapter();
+        if (!operationId) {
+            const collectionImportOperation = await findOrCreateOperation(adapter, db.$id, "importData", collection.$id);
+            // Store the operation ID in the map
+            this.collectionImportOperations.set(this.getCollectionKey(collection.name), collectionImportOperation.$id);
+            operationId = collectionImportOperation.$id;
+        }
+        if (operationId) {
+            await updateOperation(adapter, db.$id, operationId, {
+                status: "in_progress",
+                total: rawData.length,
+            });
+        }
+        // Retrieve the current user data and the current collection data from the import map
+        const currentUserData = this.importMap.get(this.getCollectionKey("users"));
+        const currentData = this.importMap.get(this.getCollectionKey(collection.name));
+        // Log errors if the necessary data is not found in the import map
+        if (!currentUserData) {
+            logger.error(`No data found for collection ${"users"} for createDef but it says it's supposed to have one...`);
+            return;
+        }
+        else if (!currentData) {
+            logger.error(`No data found for collection ${collection.name} for createDef but it says it's supposed to have one...`);
+            return;
+        }
+        // Iterate through each item in the raw data
+        for (const item of rawData) {
+            // Prepare user data, check for duplicates, and remove user-specific keys
+            let { transformedItem, existingId, userData } = this.prepareUserData(item, importDef.attributeMappings, importDef.primaryKeyField, this.getTrueUniqueId(this.getCollectionKey("users")));
+            logger.info(`In create user -- transformedItem: ${JSON.stringify(transformedItem, null, 2)}`);
+            // Generate a new unique ID for the item or use existing ID
+            if (!existingId && !userData.finalData?.userId) {
+                // No existing user ID, generate a new unique ID
+                existingId = this.getTrueUniqueId(this.getCollectionKey(collection.name));
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
+            }
+            else if (!existingId && userData.finalData?.userId) {
+                // Existing user ID, use it as the new ID
+                existingId = userData.finalData.userId;
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
+            }
+            // Create a context object for the item, including the new ID
+            let context = this.createContext(db, collection, item, existingId);
+            // Merge the transformed data into the context
+            context = { ...context, ...transformedItem, ...userData.finalData };
+            // If a primary key field is defined, handle the ID mapping and check for duplicates
+            if (importDef.primaryKeyField) {
+                const oldId = item[importDef.primaryKeyField];
+                // Check if the oldId already exists to handle potential duplicates
+                if (this.oldIdToNewIdPerCollectionMap
+                    .get(this.getCollectionKey(collection.name))
+                    ?.has(`${oldId}`)) {
+                    // Found a duplicate oldId, now decide how to merge or handle these duplicates
+                    for (const data of currentData.data) {
+                        if (data.finalData.docId === oldId ||
+                            data.finalData.userId === oldId ||
+                            data.context.docId === oldId ||
+                            data.context.userId === oldId) {
+                            transformedItem = this.mergeObjects(data.finalData, transformedItem);
+                        }
+                    }
+                }
+                else {
+                    // No duplicate found, simply map the oldId to the new itemId
+                    collectionOldIdToNewIdMap?.set(`${oldId}`, `${existingId}`);
+                }
+            }
+            // Handle merging for currentUserData
+            for (let i = 0; i < currentUserData.data.length; i++) {
+                const currentUserDataItem = currentUserData.data[i];
+                const samePhones = currentUserDataItem.finalData.phone &&
+                    transformedItem.phone &&
+                    currentUserDataItem.finalData.phone === transformedItem.phone;
+                const sameEmails = currentUserDataItem.finalData.email &&
+                    transformedItem.email &&
+                    currentUserDataItem.finalData.email === transformedItem.email;
+                if ((currentUserDataItem.finalData.docId === existingId ||
+                    currentUserDataItem.finalData.userId === existingId) &&
+                    (samePhones || sameEmails) &&
+                    currentUserDataItem.finalData &&
+                    userData.finalData) {
+                    const userDataMerged = this.mergeObjects(currentUserData.data[i].finalData, userData.finalData);
+                    currentUserData.data[i].finalData = userDataMerged;
+                    this.importMap.set(this.getCollectionKey("users"), currentUserData);
+                }
+            }
+            // Update the attribute mappings with any actions that need to be performed post-import
+            // We added the basePath to get the folder from the filePath
+            const mappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, context, transformedItem);
+            // Update the import definition with the new attribute mappings
+            const newImportDef = {
+                ...importDef,
+                attributeMappings: mappingsWithActions,
+            };
+            const updatedData = this.importMap.get(this.getCollectionKey(collection.name));
+            let foundData = false;
+            for (let i = 0; i < updatedData.data.length; i++) {
+                if (updatedData.data[i].finalData.docId === existingId ||
+                    updatedData.data[i].finalData.userId === existingId ||
+                    updatedData.data[i].context.docId === existingId ||
+                    updatedData.data[i].context.userId === existingId) {
+                    updatedData.data[i].finalData = this.mergeObjects(updatedData.data[i].finalData, transformedItem);
+                    updatedData.data[i].context = this.mergeObjects(updatedData.data[i].context, context);
+                    const mergedImportDef = {
+                        ...updatedData.data[i].importDef,
+                        idMappings: [
+                            ...(updatedData.data[i].importDef?.idMappings || []),
+                            ...(newImportDef.idMappings || []),
+                        ],
+                        attributeMappings: [
+                            ...(updatedData.data[i].importDef?.attributeMappings || []),
+                            ...(newImportDef.attributeMappings || []),
+                        ],
+                    };
+                    updatedData.data[i].importDef = mergedImportDef;
+                    this.importMap.set(this.getCollectionKey(collection.name), updatedData);
+                    this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
+                    foundData = true;
+                }
+            }
+            if (!foundData) {
+                // Add new data to the associated collection
+                updatedData.data.push({
+                    rawData: item,
+                    context: context,
+                    importDef: newImportDef,
+                    finalData: transformedItem,
+                });
+                this.importMap.set(this.getCollectionKey(collection.name), updatedData);
+                this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
+            }
+        }
+    }
+    /**
+     * Prepares the data for creating documents in a collection.
+     * This involves loading the data, transforming it, and handling ID mappings.
+     *
+     * @param db - The database configuration.
+     * @param collection - The collection configuration.
+     * @param importDef - The import definition containing the attribute mappings and other relevant info.
+     */
+    async prepareCreateData(db, collection, importDef) {
+        // Load the raw data based on the import definition
+        const rawData = this.loadData(importDef);
+        let operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
+        const adapter = await this.getAdapter();
+        if (!operationId) {
+            const collectionImportOperation = await findOrCreateOperation(adapter, db.$id, "importData", collection.$id);
+            // Store the operation ID in the map
+            this.collectionImportOperations.set(this.getCollectionKey(collection.name), collectionImportOperation.$id);
+            operationId = collectionImportOperation.$id;
+        }
+        if (operationId) {
+            await updateOperation(adapter, db.$id, operationId, {
+                status: "in_progress",
+                total: rawData.length,
+            });
+        }
+        // Initialize a new map for old ID to new ID mappings
+        const oldIdToNewIdMapNew = new Map();
+        // Retrieve or initialize the collection-specific old ID to new ID map
+        const collectionOldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(this.getCollectionKey(collection.name)) ||
+            this.oldIdToNewIdPerCollectionMap
+                .set(this.getCollectionKey(collection.name), oldIdToNewIdMapNew)
+                .get(this.getCollectionKey(collection.name));
+        const isRegions = collection.name.toLowerCase() === "regions";
+        // Iterate through each item in the raw data
+        for (const item of rawData) {
+            // Generate a new unique ID for the item
+            const itemIdNew = this.getTrueUniqueId(this.getCollectionKey(collection.name));
+            if (isRegions) {
+                logger.info(`Creating region: ${JSON.stringify(item, null, 2)}`);
+            }
+            // Retrieve the current collection data from the import map
+            const currentData = this.importMap.get(this.getCollectionKey(collection.name));
+            // Create a context object for the item, including the new ID
+            let context = this.createContext(db, collection, item, itemIdNew);
+            // Transform the item data based on the attribute mappings
+            let transformedData = this.transformData(item, importDef.attributeMappings);
+            // If a primary key field is defined, handle the ID mapping and check for duplicates
+            if (importDef.primaryKeyField) {
+                const oldId = item[importDef.primaryKeyField];
+                if (collectionOldIdToNewIdMap?.has(`${oldId}`)) {
+                    logger.error(`Collection ${collection.name} has multiple documents with the same primary key ${oldId}`);
+                    continue;
+                }
+                collectionOldIdToNewIdMap?.set(`${oldId}`, `${itemIdNew}`);
+            }
+            // Merge the transformed data into the context
+            context = { ...context, ...transformedData };
+            // Validate the item before proceeding
+            const isValid = this.importDataActions.validateItem(transformedData, importDef.attributeMappings, context);
+            if (!isValid) {
+                continue;
+            }
+            // Update the attribute mappings with any actions that need to be performed post-import
+            // We added the basePath to get the folder from the filePath
+            const mappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, context, transformedData);
+            // Update the import definition with the new attribute mappings
+            const newImportDef = {
+                ...importDef,
+                attributeMappings: mappingsWithActions,
+            };
+            // If the current collection data exists, add the item with its context and final data
+            if (currentData && currentData.data) {
+                currentData.data.push({
+                    rawData: item,
+                    context: context,
+                    importDef: newImportDef,
+                    finalData: transformedData,
+                });
+                this.importMap.set(this.getCollectionKey(collection.name), currentData);
+                this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
+            }
+            else {
+                logger.error(`No data found for collection ${collection.name} for createDef but it says it's supposed to have one...`);
+                continue;
+            }
+        }
+    }
+    /**
+     * Prepares the data for updating documents within a collection.
+     * This method loads the raw data based on the import definition, transforms it according to the attribute mappings,
+     * finds the new ID for each item based on the primary key or update mapping, and then validates the transformed data.
+     * If the data is valid, it updates the import definition with any post-import actions and adds the item to the current collection data.
+     *
+     * @param db - The database configuration.
+     * @param collection - The collection configuration.
+     * @param importDef - The import definition containing the attribute mappings and other relevant info.
+     */
+    async prepareUpdateData(db, collection, importDef) {
+        const currentData = this.importMap.get(this.getCollectionKey(collection.name));
+        const oldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(this.getCollectionKey(collection.name));
+        if (!(currentData?.data && currentData?.data.length > 0) &&
+            !oldIdToNewIdMap) {
+            logger.error(`No data found for collection ${collection.name} for updateDef but it says it's supposed to have one...`);
+            return;
+        }
+        const rawData = this.loadData(importDef);
+        const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
+        if (!operationId) {
+            throw new Error(`No import operation found for collection ${collection.name}`);
+        }
+        for (const item of rawData) {
+            let transformedData = this.transformData(item, importDef.attributeMappings);
+            let newId;
+            let oldId;
+            let itemDataToUpdate;
+            // Try to find itemDataToUpdate using updateMapping
+            if (importDef.updateMapping) {
+                oldId =
+                    item[importDef.updateMapping.originalIdField] ||
+                        transformedData[importDef.updateMapping.originalIdField];
+                if (oldId) {
+                    itemDataToUpdate = currentData?.data.find(({ context, finalData }) => {
+                        const targetField = importDef.updateMapping.targetField ??
+                            importDef.updateMapping.originalIdField;
+                        return (`${context[targetField]}` === `${oldId}` ||
+                            `${finalData[targetField]}` === `${oldId}`);
+                    });
+                    if (itemDataToUpdate) {
+                        newId = itemDataToUpdate.context.docId;
+                    }
+                }
+            }
+            // If updateMapping is not defined or did not find the item, use primaryKeyField
+            if (!itemDataToUpdate && importDef.primaryKeyField) {
+                oldId =
+                    item[importDef.primaryKeyField] ||
+                        transformedData[importDef.primaryKeyField];
+                if (oldId && oldId.length > 0) {
+                    newId = oldIdToNewIdMap?.get(`${oldId}`);
+                    if (!newId &&
+                        this.getCollectionKey(this.config.usersCollectionName) ===
+                            this.getCollectionKey(collection.name)) {
+                        for (const [key, value] of this.mergedUserMap.entries()) {
+                            if (value.includes(`${oldId}`)) {
+                                newId = key;
+                                break;
+                            }
+                        }
+                    }
+                }
+                if (oldId && !itemDataToUpdate) {
+                    itemDataToUpdate = currentData?.data.find((data) => `${data.context[importDef.primaryKeyField]}` === `${oldId}`);
+                }
+            }
+            if (!oldId) {
+                logger.error(`No old ID found (to update another document with) in prepareUpdateData for ${collection.name}, ${JSON.stringify(item, null, 2)}`);
+                continue;
+            }
+            if (!newId && !itemDataToUpdate) {
+                logger.error(`No new id && no data found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
+                continue;
+            }
+            else if (itemDataToUpdate) {
+                newId =
+                    itemDataToUpdate.finalData.docId || itemDataToUpdate.context.docId;
+                if (!newId) {
+                    logger.error(`No new id found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but has itemDataToUpdate ${JSON.stringify(itemDataToUpdate, null, 2)} but it says it's supposed to have one...`);
+                    continue;
+                }
+            }
+            if (!itemDataToUpdate || !newId) {
+                logger.error(`No data or ID (docId) found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
+                continue;
+            }
+            transformedData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
+            // Create a context object for the item, including the new ID and transformed data
+            let context = itemDataToUpdate.context;
+            context = this.mergeObjects(context, transformedData);
+            // Validate the item before proceeding
+            const isValid = this.importDataActions.validateItem(item, importDef.attributeMappings, context);
+            if (!isValid) {
+                logger.info(`Skipping item: ${JSON.stringify(item, null, 2)} because it's invalid`);
+                continue;
+            }
+            // Update the attribute mappings with any actions that need to be performed post-import
+            // We added the basePath to get the folder from the filePath
+            const mappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, context, transformedData);
+            // Update the import definition with the new attribute mappings
+            const newImportDef = {
+                ...importDef,
+                attributeMappings: mappingsWithActions,
+            };
+            if (itemDataToUpdate) {
+                itemDataToUpdate.finalData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
+                itemDataToUpdate.context = context;
+                // Fix: Ensure we properly merge the attribute mappings and their actions
+                const mergedAttributeMappings = newImportDef.attributeMappings.map((newMapping) => {
+                    const existingMapping = itemDataToUpdate.importDef?.attributeMappings.find((m) => m.targetKey === newMapping.targetKey);
+                    return {
+                        ...newMapping,
+                        postImportActions: [
+                            ...(existingMapping?.postImportActions || []),
+                            ...(newMapping.postImportActions || []),
+                        ],
+                    };
+                });
+                itemDataToUpdate.importDef = {
+                    ...newImportDef,
+                    attributeMappings: mergedAttributeMappings,
+                };
+                // Debug logging
+                if (mergedAttributeMappings.some((m) => m.postImportActions?.length > 0)) {
+                    logger.info(`Post-import actions for ${collection.name}: ${JSON.stringify(mergedAttributeMappings
+                        .filter((m) => m.postImportActions?.length > 0)
+                        .map((m) => ({
+                        targetKey: m.targetKey,
+                        actions: m.postImportActions,
+                    })), null, 2)}`);
+                }
+            }
+            else {
+                currentData.data.push({
+                    rawData: item,
+                    context: context,
+                    importDef: newImportDef,
+                    finalData: transformedData,
+                });
+            }
+            // Since we're modifying currentData in place, we ensure no duplicates are added
+            this.importMap.set(this.getCollectionKey(collection.name), currentData);
+        }
+    }
+    updateReferencesBasedOnAttributeMappings() {
+        if (!this.config.collections) {
+            return;
+        }
+        this.config.collections.forEach((collectionConfig) => {
+            const collectionName = collectionConfig.name;
+            const collectionData = this.importMap.get(this.getCollectionKey(collectionName));
+            if (!collectionData) {
+                logger.error(`No data found for collection ${collectionName}`);
+                return;
+            }
+            collectionData.data.forEach((dataItem) => {
+                collectionConfig.importDefs.forEach((importDef) => {
+                    if (!importDef.idMappings)
+                        return; // Skip collections without idMappings
+                    importDef.idMappings.forEach((mapping) => {
+                        if (mapping && mapping.targetField) {
+                            const idsToUpdate = Array.isArray(dataItem[mapping.targetField])
+                                ? dataItem[mapping.targetField]
+                                : [dataItem[mapping.targetField]];
+                            const updatedIds = idsToUpdate.map((id) => this.getMergedId(id, mapping.targetCollection));
+                            // Update the dataItem with the new IDs
+                            dataItem[mapping.targetField] =
+                                Array.isArray(dataItem[mapping.targetField])
+                                    ? updatedIds
+                                    : updatedIds[0];
+                        }
+                    });
+                });
+            });
+        });
+    }
+    getMergedId(oldId, relatedCollectionName) {
+        // Retrieve the old to new ID map for the related collection
+        const oldToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(this.getCollectionKey(relatedCollectionName));
+        // If there's a mapping for the old ID, return the new ID
+        if (oldToNewIdMap && oldToNewIdMap.has(`${oldId}`)) {
+            return oldToNewIdMap.get(`${oldId}`); // The non-null assertion (!) is used because we checked if the map has the key
+        }
+        // If no mapping is found, return the old ID as a fallback
+        return oldId;
+    }
+    /**
+     * Generates attribute mappings with post-import actions based on the provided attribute mappings.
+     * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
+     * and update the field with the file's ID if necessary.
+     *
+     * @param attributeMappings - The attribute mappings from the import definition.
+     * @param context - The context object containing information about the database, collection, and document.
+     * @param item - The item being imported, used for resolving template paths in fileData mappings.
+     * @returns The attribute mappings updated with any necessary post-import actions.
+     */
+    getAttributeMappingsWithActions(attributeMappings, context, item) {
+        // Iterate over each attribute mapping to check for fileData attributes
+        return attributeMappings.map((mapping) => {
+            if (mapping.fileData) {
+                // Resolve the file path using the provided template, context, and item
+                let mappingFilePath = this.importDataActions.resolveTemplate(mapping.fileData.path, context, item);
+                // Ensure the file path is absolute if it doesn't start with "http"
+                if (!mappingFilePath.toLowerCase().startsWith("http")) {
+                    // First try the direct path
+                    let fullPath = path.resolve(this.appwriteFolderPath, mappingFilePath);
+                    // If file doesn't exist, search in subdirectories
+                    if (!fs.existsSync(fullPath)) {
+                        const findFileInDir = (dir) => {
+                            const files = fs.readdirSync(dir);
+                            for (const file of files) {
+                                const filePath = path.join(dir, file);
+                                const stat = fs.statSync(filePath);
+                                if (stat.isDirectory()) {
+                                    // Recursively search subdirectories
+                                    const found = findFileInDir(filePath);
+                                    if (found)
+                                        return found;
+                                }
+                                else if (file === path.basename(mappingFilePath)) {
+                                    return filePath;
+                                }
+                            }
+                            return null;
+                        };
+                        const foundPath = findFileInDir(this.appwriteFolderPath);
+                        if (foundPath) {
+                            mappingFilePath = foundPath;
+                        }
+                        else {
+                            logger.warn(`File not found in any subdirectory: ${mappingFilePath}`);
+                            // Keep the original resolved path as fallback
+                            mappingFilePath = fullPath;
+                        }
+                    }
+                    else {
+                        mappingFilePath = fullPath;
+                    }
+                }
+                // Define the after-import action to create a file and update the field
+                const afterImportAction = {
+                    action: "createFileAndUpdateField",
+                    params: [
+                        "{dbId}",
+                        "{collId}",
+                        "{docId}",
+                        mapping.targetKey,
+                        `${this.config.documentBucketId}_${context.dbName
+                            .toLowerCase()
+                            .replace(" ", "")}`, // Assuming 'images' is your bucket ID
+                        mappingFilePath,
+                        mapping.fileData.name,
+                    ],
+                };
+                // Add the after-import action to the mapping's postImportActions array
+                const postImportActions = mapping.postImportActions
+                    ? [...mapping.postImportActions, afterImportAction]
+                    : [afterImportAction];
+                return { ...mapping, postImportActions };
+            }
+            // Return the mapping unchanged if no fileData attribute is found
+            return mapping;
+        });
+    }
+}