appwrite-utils-cli 0.0.54 → 0.0.57

package/README.md

@@ -132,6 +132,8 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 ### Changelog
 
+- 0.0.57: Fixed `dataLoader`'s `idMapping`'s giving me issues
+- 0.0.55: Added `documentExists` check to batch creation functionality to try to prevent duplicates
 - 0.0.54: Various fixes in here
 - 0.0.50: Actually fixed the slight bug, it was really in the `mergeObjects`
 - 0.0.49: Fixed a slight bug with `dataLoader` not mapping updates correctly with `updateMapping`

package/dist/migrations/afterImportActions.js

@@ -3,7 +3,7 @@ import path from "path";
 import fs from "fs";
 import os from "os";
 import { logger } from "./logging.js";
-import {} from "appwrite-utils";
+import { tryAwaitWithRetry, } from "appwrite-utils";
 export const getDatabaseFromConfig = (config) => {
     if (!config.appwriteClient) {
         config.appwriteClient = new Client()
@@ -242,7 +242,7 @@ export const afterImportActions = {
             // Define a helper function to check if a value is a URL
             const isUrl = (value) => typeof value === "string" &&
                 (value.startsWith("http://") || value.startsWith("https://"));
-            const doc = await db.getDocument(dbId, collId, docId);
+            const doc = await tryAwaitWithRetry(async () => await db.getDocument(dbId, collId, docId));
             const existingFieldValue = doc[fieldName];
             // Handle the case where the field is an array
             let updateData = isArray ? [] : "";
@@ -265,7 +265,7 @@ export const afterImportActions = {
                 // Create InputFile from the downloaded file
                 const inputFile = InputFile.fromPath(tempFilePath, fileName);
                 // Use the full file name (with extension) for creating the file
-                const file = await storage.createFile(bucketId, ID.unique(), inputFile);
+                const file = await tryAwaitWithRetry(async () => await storage.createFile(bucketId, ID.unique(), inputFile));
                 console.log("Created file from URL: ", file.$id);
                 // After uploading, adjust the updateData based on whether the field is an array or not
                 if (isArray) {
@@ -274,15 +274,9 @@ export const afterImportActions = {
                 else {
                     updateData = file.$id; // Set the new file ID
                 }
-                await db.updateDocument(dbId, collId, doc.$id, {
+                await tryAwaitWithRetry(async () => await db.updateDocument(dbId, collId, doc.$id, {
                     [fieldName]: updateData,
-                });
-                // console.log(
-                //   "Updating document with file: ",
-                //   doc.$id,
-                //   `${fieldName}: `,
-                //   updateData
-                // );
+                }));
                 // If the file was downloaded, delete it after uploading
                 fs.unlinkSync(tempFilePath);
             }
@@ -295,16 +289,16 @@ export const afterImportActions = {
                 }
                 const pathToFile = path.join(filePath, fileFullName);
                 const inputFile = InputFile.fromPath(pathToFile, fileName);
-                const file = await storage.createFile(bucketId, ID.unique(), inputFile);
+                const file = await tryAwaitWithRetry(async () => await storage.createFile(bucketId, ID.unique(), inputFile));
                 if (isArray) {
                     updateData = [...updateData, file.$id]; // Append the new file ID
                 }
                 else {
                     updateData = file.$id; // Set the new file ID
                 }
-                await db.updateDocument(dbId, collId, doc.$id, {
+                tryAwaitWithRetry(async () => await db.updateDocument(dbId, collId, doc.$id, {
                     [fieldName]: updateData,
-                });
+                }));
                 console.log("Created file from path: ", file.$id);
             }
         }

package/dist/migrations/dataLoader.d.ts

@@ -1748,6 +1748,14 @@ export declare class DataLoader {
      *
      * @return {void} This function does not return anything.
      */
+    /**
+     * Gets the value to match for a given key in the final data or context.
+     * @param finalData - The final data object.
+     * @param context - The context object.
+     * @param key - The key to get the value for.
+     * @returns The value to match for from finalData or Context
+     */
+    getValueFromData(finalData: any, context: any, key: string): any;
     updateOldReferencesForNew(): void;
     private writeMapsToJsonFile;
     /**

package/dist/migrations/dataLoader.js

@@ -108,9 +108,9 @@ export class DataLoader {
                     result[key] = [...value, updateValue].filter((item) => item !== null && item !== undefined && item !== "");
                 }
             }
-            else if (typeof value === "object") {
+            else if (typeof value === "object" && !Array.isArray(value)) {
                 // If the value is an object, we need to merge it
-                if (typeof update[key] === "object") {
+                if (typeof update[key] === "object" && !Array.isArray(update[key])) {
                     result[key] = this.mergeObjects(value, update[key]);
                 }
             }
@@ -127,7 +127,10 @@ export class DataLoader {
             else if (!Object.hasOwn(source, key)) {
                 result[key] = value;
             }
-            else if (typeof source[key] === "object" && typeof value === "object") {
+            else if (typeof source[key] === "object" &&
+                typeof value === "object" &&
+                !Array.isArray(source[key]) &&
+                !Array.isArray(value)) {
                 result[key] = this.mergeObjects(source[key], value);
             }
             else if (Array.isArray(source[key]) && Array.isArray(value)) {
@@ -207,9 +210,6 @@ export class DataLoader {
     transformData(item, attributeMappings) {
         // Convert the item using the attribute mappings provided
         const convertedItem = convertObjectByAttributeMappings(item, attributeMappings);
-        if (item["region"]) {
-            logger.info(`Converted item: ${JSON.stringify(convertedItem, null, 2)}`);
-        }
         // Run additional converter functions on the converted item, if any
         return this.importDataActions.runConverterFunctions(convertedItem, attributeMappings);
     }
@@ -315,12 +315,10 @@ export class DataLoader {
                 const updateDefs = collection.importDefs.filter((def) => def.type === "update");
                 for (const createDef of createDefs) {
                     if (!isUsersCollection) {
-                        console.log(`${collection.name} is not users collection`);
                         await this.prepareCreateData(db, collection, createDef);
                     }
                     else {
                         // Special handling for users collection if needed
-                        console.log(`${collection.name} is users collection`);
                         await this.prepareUserCollectionCreateData(db, collection, createDef);
                     }
                 }
@@ -396,6 +394,21 @@ export class DataLoader {
     //     }
     //   }
     // }
+    /**
+     * Gets the value to match for a given key in the final data or context.
+     * @param finalData - The final data object.
+     * @param context - The context object.
+     * @param key - The key to get the value for.
+     * @returns The value to match for from finalData or Context
+     */
+    getValueFromData(finalData, context, key) {
+        if (context[key] !== undefined &&
+            context[key] !== null &&
+            context[key] !== "") {
+            return context[key];
+        }
+        return finalData[key];
+    }
     updateOldReferencesForNew() {
         if (!this.config.collections) {
             return;
@@ -416,7 +429,7 @@ export class DataLoader {
                                 const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
                                 const fieldToSetKey = idMapping.fieldToSet || idMapping.sourceField;
                                 const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
-                                const valueToMatch = collectionData.data[i].context[idMapping.sourceField];
+                                const valueToMatch = this.getValueFromData(collectionData.data[i].finalData, collectionData.data[i].context, idMapping.sourceField);
                                 // Skip if value to match is missing or empty
                                 if (!valueToMatch ||
                                     _.isEmpty(valueToMatch) ||
@@ -427,8 +440,8 @@ export class DataLoader {
                                 if (!targetCollectionData || !targetCollectionData.data)
                                     continue;
                                 // Find matching data in the target collection
-                                const foundData = targetCollectionData.data.filter(({ context }) => {
-                                    const targetValue = context[targetFieldKey];
+                                const foundData = targetCollectionData.data.filter(({ context, finalData }) => {
+                                    const targetValue = this.getValueFromData(finalData, context, targetFieldKey);
                                     const isMatch = `${targetValue}` === `${valueToMatch}`;
                                     // Ensure the targetValue is defined and not null
                                     return (isMatch &&
@@ -438,7 +451,7 @@ export class DataLoader {
                                 // Log and skip if no matching data found
                                 if (!foundData.length) {
                                     console.log(`No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`);
-                                    logger.error(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
+                                    logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
                                     continue;
                                 }
                                 needsUpdate = true;
@@ -451,7 +464,10 @@ export class DataLoader {
                                 // Get the current data to be updated
                                 const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
                                 // Extract the new data to set
-                                const newData = foundData.map((data) => data.context[idMapping.targetField]);
+                                const newData = foundData.map((data) => {
+                                    const valueFound = this.getValueFromData(data.finalData, data.context, idMapping.targetField);
+                                    return valueFound;
+                                });
                                 // Handle cases where current data is an array
                                 if (isFieldToSetArray) {
                                     if (!currentDataFiltered) {
@@ -460,15 +476,23 @@ export class DataLoader {
                                             Array.isArray(newData) ? newData : [newData];
                                     }
                                     else {
-                                        // Merge arrays if new data is non-empty array and filter for uniqueness
-                                        collectionData.data[i].finalData[fieldToSetKey] = [
-                                            ...new Set([
-                                                ...(Array.isArray(currentDataFiltered)
-                                                    ? currentDataFiltered
-                                                    : [currentDataFiltered]),
-                                                ...newData,
-                                            ].filter((value) => `${value}` !== `${valueToMatch}`)),
-                                        ];
+                                        if (Array.isArray(currentDataFiltered)) {
+                                            // Convert current data to array and merge if new data is non-empty array
+                                            collectionData.data[i].finalData[fieldToSetKey] = [
+                                                ...new Set([...currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}` && value)),
+                                            ];
+                                        }
+                                        else {
+                                            // Merge arrays if new data is non-empty array and filter for uniqueness
+                                            collectionData.data[i].finalData[fieldToSetKey] = [
+                                                ...new Set([
+                                                    ...(Array.isArray(currentDataFiltered)
+                                                        ? currentDataFiltered
+                                                        : [currentDataFiltered]),
+                                                    ...newData,
+                                                ].filter((value) => `${value}` !== `${valueToMatch}` && value)),
+                                            ];
+                                        }
                                     }
                                 }
                                 else {
@@ -481,7 +505,7 @@ export class DataLoader {
                                         // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
                                         // and take the first value, because it's an array and the attribute is not an array
                                         collectionData.data[i].finalData[fieldToSetKey] = [
-                                            ...new Set([currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}`)),
+                                            ...new Set([currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}` && value)),
                                         ].slice(0, 1)[0];
                                     }
                                     else if (!Array.isArray(newData) && newData !== undefined) {
@@ -501,39 +525,45 @@ export class DataLoader {
         }
     }
     writeMapsToJsonFile() {
-        const outputDir = path.resolve(process.cwd());
-        const outputFile = path.join(outputDir, "dataLoaderOutput.json");
-        const dataToWrite = {
-            // Convert Maps to arrays of entries for serialization
-            oldIdToNewIdPerCollectionMap: Array.from(this.oldIdToNewIdPerCollectionMap.entries()).map(([key, value]) => {
-                return {
-                    collection: key,
-                    data: Array.from(value.entries()),
-                };
-            }),
-            mergedUserMap: Array.from(this.mergedUserMap.entries()),
-            dataFromCollections: Array.from(this.importMap.entries()).map(([key, value]) => {
-                return {
-                    collection: key,
-                    data: value.data.map((item) => item.finalData),
-                };
-            }),
-            // emailToUserIdMap: Array.from(this.emailToUserIdMap.entries()),
-            // phoneToUserIdMap: Array.from(this.phoneToUserIdMap.entries()),
-        };
-        // Use JSON.stringify with a replacer function to handle Maps
-        const replacer = (key, value) => {
-            if (value instanceof Map) {
-                return Array.from(value.entries());
-            }
-            return value;
+        const outputDir = path.resolve(process.cwd(), "zlogs");
+        // Ensure the logs directory exists
+        if (!fs.existsSync(outputDir)) {
+            fs.mkdirSync(outputDir);
+        }
+        // Helper function to write data to a file
+        const writeToFile = (fileName, data) => {
+            const outputFile = path.join(outputDir, fileName);
+            fs.writeFile(outputFile, JSON.stringify(data, null, 2), "utf8", (err) => {
+                if (err) {
+                    console.error(`Error writing data to ${fileName}:`, err);
+                    return;
+                }
+                console.log(`Data successfully written to ${fileName}`);
+            });
         };
-        fs.writeFile(outputFile, JSON.stringify(dataToWrite, replacer, 2), "utf8", (err) => {
-            if (err) {
-                console.error("Error writing data to JSON file:", err);
-                return;
-            }
-            console.log(`Data successfully written to ${outputFile}`);
+        // Convert Maps to arrays of entries for serialization
+        const oldIdToNewIdPerCollectionMap = Array.from(this.oldIdToNewIdPerCollectionMap.entries()).map(([key, value]) => {
+            return {
+                collection: key,
+                data: Array.from(value.entries()),
+            };
+        });
+        const mergedUserMap = Array.from(this.mergedUserMap.entries());
+        // Write each part to a separate file
+        writeToFile("oldIdToNewIdPerCollectionMap.json", oldIdToNewIdPerCollectionMap);
+        writeToFile("mergedUserMap.json", mergedUserMap);
+        // Write each collection's data to a separate file
+        this.importMap.forEach((value, key) => {
+            const data = {
+                collection: key,
+                data: value.data.map((item) => {
+                    return {
+                        finalData: item.finalData,
+                        context: item.context,
+                    };
+                }),
+            };
+            writeToFile(`${key}.json`, data);
         });
     }
     /**
@@ -557,9 +587,15 @@ export class DataLoader {
         // Check for duplicate email and phone
         if (email && this.emailToUserIdMap.has(email)) {
             existingId = this.emailToUserIdMap.get(email);
+            if (phone && !this.phoneToUserIdMap.has(phone)) {
+                this.phoneToUserIdMap.set(phone, newId);
+            }
         }
         else if (phone && this.phoneToUserIdMap.has(phone)) {
             existingId = this.phoneToUserIdMap.get(phone);
+            if (email && !this.emailToUserIdMap.has(email)) {
+                this.emailToUserIdMap.set(email, newId);
+            }
         }
         else {
             if (email)
@@ -654,7 +690,6 @@ export class DataLoader {
             this.oldIdToNewIdPerCollectionMap
                 .set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
                 .get(this.getCollectionKey(collection.name));
-        console.log(`${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`);
         if (!operationId) {
             throw new Error(`No import operation found for collection ${collection.name}`);
         }
@@ -740,7 +775,10 @@ export class DataLoader {
                 if (currentData.data[i].finalData.docId === existingId ||
                     currentData.data[i].finalData.userId === existingId) {
                     currentData.data[i].finalData = this.mergeObjects(currentData.data[i].finalData, transformedItem);
-                    currentData.data[i].context = context;
+                    currentData.data[i].context = {
+                        ...currentData.data[i].context,
+                        ...context,
+                    };
                     currentData.data[i].importDef = newImportDef;
                     this.importMap.set(this.getCollectionKey(collection.name), currentData);
                     this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
@@ -786,7 +824,6 @@ export class DataLoader {
             this.oldIdToNewIdPerCollectionMap
                 .set(this.getCollectionKey(collection.name), oldIdToNewIdMapNew)
                 .get(this.getCollectionKey(collection.name));
-        console.log(`${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`);
         const isRegions = collection.name.toLowerCase() === "regions";
         // Iterate through each item in the raw data
         for (const item of rawData) {
@@ -801,6 +838,9 @@ export class DataLoader {
             let context = this.createContext(db, collection, item, itemIdNew);
             // Transform the item data based on the attribute mappings
             const transformedData = this.transformData(item, importDef.attributeMappings);
+            if (collection.name.toLowerCase() === "councils") {
+                console.log("Transformed Council: ", transformedData);
+            }
             if (isRegions) {
                 logger.info(`Transformed region: ${JSON.stringify(transformedData, null, 2)}`);
             }
@@ -874,7 +914,6 @@ export class DataLoader {
             let itemDataToUpdate;
             // Try to find itemDataToUpdate using updateMapping
             if (importDef.updateMapping) {
-                console.log(importDef.updateMapping);
                 oldId =
                     item[importDef.updateMapping.originalIdField] ||
                         transformedData[importDef.updateMapping.originalIdField];
@@ -898,7 +937,7 @@ export class DataLoader {
                 oldId =
                     item[importDef.primaryKeyField] ||
                         transformedData[importDef.primaryKeyField];
-                if (oldId) {
+                if (oldId && oldId.length > 0) {
                     newId = oldIdToNewIdMap?.get(`${oldId}`);
                     if (!newId &&
                         this.getCollectionKey(this.config.usersCollectionName) ===

package/dist/migrations/importController.js

@@ -7,7 +7,7 @@ import { logger } from "./logging.js";
 import { updateOperation } from "./migrationHelper.js";
 import { BatchSchema, OperationCreateSchema, OperationSchema, } from "./backup.js";
 import { DataLoader } from "./dataLoader.js";
-import { transferDocumentsBetweenDbsLocalToLocal } from "./collections.js";
+import { documentExists, transferDocumentsBetweenDbsLocalToLocal, } from "./collections.js";
 import { transferDatabaseLocalToLocal } from "./databases.js";
 import { transferStorageLocalToLocal } from "./storage.js";
 export class ImportController {
@@ -166,7 +166,32 @@ export class ImportController {
             for (let i = 0; i < dataSplit.length; i++) {
                 const batches = dataSplit[i];
                 console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
-                const batchPromises = batches.map((item) => {
+                // const documentExistsPromises = batches.map(async (item) => {
+                //   try {
+                //     const id =
+                //       item.finalData.docId ||
+                //       item.finalData.userId ||
+                //       item.context.docId ||
+                //       item.context.userId;
+                //     if (!item.finalData) {
+                //       return Promise.resolve(null);
+                //     }
+                //     return tryAwaitWithRetry(
+                //       async () =>
+                //         await documentExists(
+                //           this.database,
+                //           db.$id,
+                //           collection.$id,
+                //           item.finalData
+                //         )
+                //     );
+                //   } catch (error) {
+                //     console.error(error);
+                //     return Promise.resolve(null);
+                //   }
+                // });
+                // const documentExistsResults = await Promise.all(documentExistsPromises);
+                const batchPromises = batches.map((item, index) => {
                     try {
                         const id = item.finalData.docId ||
                             item.finalData.userId ||

package/dist/migrations/logging.js

@@ -1,15 +1,27 @@
 import winston from "winston";
+import fs from "fs";
+import path from "path";
+// Ensure the logs directory exists
+const logDir = path.join(process.cwd(), "zlogs");
+if (!fs.existsSync(logDir)) {
+    fs.mkdirSync(logDir);
+}
 export const logger = winston.createLogger({
-    level: "info",
+    level: "debug",
     format: winston.format.json({ space: 2 }),
     defaultMeta: { service: "appwrite-utils-cli" },
     transports: [
-        //
-        // - Write all logs with importance level of `error` or less to `error.log`
-        // - Write all logs with importance level of `info` or less to `combined.log`
-        //
-        new winston.transports.File({ filename: "error.log", level: "error" }),
-        new winston.transports.File({ filename: "warn.log", level: "warn" }),
-        new winston.transports.File({ filename: "combined.log" }),
+        new winston.transports.File({
+            filename: path.join(logDir, "error.log"),
+            level: "error",
+        }),
+        new winston.transports.File({
+            filename: path.join(logDir, "warn.log"),
+            level: "warn",
+        }),
+        new winston.transports.File({
+            filename: path.join(logDir, "info.log"),
+            level: "info",
+        }),
     ],
 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.54",
+  "version": "0.0.57",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/afterImportActions.ts

@@ -11,7 +11,11 @@ import path from "path";
 import fs from "fs";
 import os from "os";
 import { logger } from "./logging.js";
-import { type AfterImportActions, type AppwriteConfig } from "appwrite-utils";
+import {
+  tryAwaitWithRetry,
+  type AfterImportActions,
+  type AppwriteConfig,
+} from "appwrite-utils";
 
 export const getDatabaseFromConfig = (config: AppwriteConfig) => {
   if (!config.appwriteClient) {
@@ -401,7 +405,9 @@ export const afterImportActions = {
         typeof value === "string" &&
         (value.startsWith("http://") || value.startsWith("https://"));
 
-      const doc = await db.getDocument(dbId, collId, docId);
+      const doc = await tryAwaitWithRetry(
+        async () => await db.getDocument(dbId, collId, docId)
+      );
       const existingFieldValue = doc[fieldName as keyof typeof doc];
 
       // Handle the case where the field is an array
@@ -432,7 +438,9 @@ export const afterImportActions = {
         const inputFile = InputFile.fromPath(tempFilePath, fileName);
 
         // Use the full file name (with extension) for creating the file
-        const file = await storage.createFile(bucketId, ID.unique(), inputFile);
+        const file = await tryAwaitWithRetry(
+          async () => await storage.createFile(bucketId, ID.unique(), inputFile)
+        );
 
         console.log("Created file from URL: ", file.$id);
 
@@ -442,15 +450,12 @@ export const afterImportActions = {
         } else {
           updateData = file.$id; // Set the new file ID
         }
-        await db.updateDocument(dbId, collId, doc.$id, {
-          [fieldName]: updateData,
-        });
-        // console.log(
-        //   "Updating document with file: ",
-        //   doc.$id,
-        //   `${fieldName}: `,
-        //   updateData
-        // );
+        await tryAwaitWithRetry(
+          async () =>
+            await db.updateDocument(dbId, collId, doc.$id, {
+              [fieldName]: updateData,
+            })
+        );
 
         // If the file was downloaded, delete it after uploading
         fs.unlinkSync(tempFilePath);
@@ -465,16 +470,21 @@ export const afterImportActions = {
         }
         const pathToFile = path.join(filePath, fileFullName);
         const inputFile = InputFile.fromPath(pathToFile, fileName);
-        const file = await storage.createFile(bucketId, ID.unique(), inputFile);
+        const file = await tryAwaitWithRetry(
+          async () => await storage.createFile(bucketId, ID.unique(), inputFile)
+        );
 
         if (isArray) {
           updateData = [...updateData, file.$id]; // Append the new file ID
         } else {
           updateData = file.$id; // Set the new file ID
         }
-        await db.updateDocument(dbId, collId, doc.$id, {
-          [fieldName]: updateData,
-        });
+        tryAwaitWithRetry(
+          async () =>
+            await db.updateDocument(dbId, collId, doc.$id, {
+              [fieldName]: updateData,
+            })
+        );
         console.log("Created file from path: ", file.$id);
       }
     } catch (error) {

package/src/migrations/dataLoader.ts

@@ -140,9 +140,9 @@ export class DataLoader {
             (item) => item !== null && item !== undefined && item !== ""
           );
         }
-      } else if (typeof value === "object") {
+      } else if (typeof value === "object" && !Array.isArray(value)) {
         // If the value is an object, we need to merge it
-        if (typeof update[key] === "object") {
+        if (typeof update[key] === "object" && !Array.isArray(update[key])) {
           result[key] = this.mergeObjects(value, update[key]);
         }
       } else {
@@ -156,7 +156,12 @@ export class DataLoader {
         continue;
       } else if (!Object.hasOwn(source, key)) {
         result[key] = value;
-      } else if (typeof source[key] === "object" && typeof value === "object") {
+      } else if (
+        typeof source[key] === "object" &&
+        typeof value === "object" &&
+        !Array.isArray(source[key]) &&
+        !Array.isArray(value)
+      ) {
         result[key] = this.mergeObjects(source[key], value);
       } else if (Array.isArray(source[key]) && Array.isArray(value)) {
         result[key] = [...new Set([...source[key], ...value])].filter(
@@ -260,9 +265,6 @@ export class DataLoader {
       item,
       attributeMappings
     );
-    if (item["region"]) {
-      logger.info(`Converted item: ${JSON.stringify(convertedItem, null, 2)}`);
-    }
     // Run additional converter functions on the converted item, if any
     return this.importDataActions.runConverterFunctions(
       convertedItem,
@@ -389,11 +391,9 @@ export class DataLoader {
         );
         for (const createDef of createDefs) {
           if (!isUsersCollection) {
-            console.log(`${collection.name} is not users collection`);
             await this.prepareCreateData(db, collection, createDef);
           } else {
             // Special handling for users collection if needed
-            console.log(`${collection.name} is users collection`);
             await this.prepareUserCollectionCreateData(
               db,
               collection,
@@ -480,6 +480,24 @@ export class DataLoader {
   //   }
   // }
 
+  /**
+   * Gets the value to match for a given key in the final data or context.
+   * @param finalData - The final data object.
+   * @param context - The context object.
+   * @param key - The key to get the value for.
+   * @returns The value to match for from finalData or Context
+   */
+  getValueFromData(finalData: any, context: any, key: string) {
+    if (
+      context[key] !== undefined &&
+      context[key] !== null &&
+      context[key] !== ""
+    ) {
+      return context[key];
+    }
+    return finalData[key];
+  }
+
   updateOldReferencesForNew() {
     if (!this.config.collections) {
       return;
@@ -510,8 +528,11 @@ export class DataLoader {
                   idMapping.fieldToSet || idMapping.sourceField;
                 const targetFieldKey =
                   idMapping.targetFieldToMatch || idMapping.targetField;
-                const valueToMatch =
-                  collectionData.data[i].context[idMapping.sourceField];
+                const valueToMatch = this.getValueFromData(
+                  collectionData.data[i].finalData,
+                  collectionData.data[i].context,
+                  idMapping.sourceField
+                );
 
                 // Skip if value to match is missing or empty
                 if (
@@ -532,8 +553,12 @@ export class DataLoader {
 
                 // Find matching data in the target collection
                 const foundData = targetCollectionData.data.filter(
-                  ({ context }) => {
-                    const targetValue = context[targetFieldKey];
+                  ({ context, finalData }) => {
+                    const targetValue = this.getValueFromData(
+                      finalData,
+                      context,
+                      targetFieldKey
+                    );
                     const isMatch = `${targetValue}` === `${valueToMatch}`;
                     // Ensure the targetValue is defined and not null
                     return (
@@ -543,13 +568,12 @@ export class DataLoader {
                     );
                   }
                 );
-
                 // Log and skip if no matching data found
                 if (!foundData.length) {
                   console.log(
                     `No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`
                   );
-                  logger.error(
+                  logger.info(
                     `No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(
                       idMapping,
                       null,
@@ -576,9 +600,14 @@ export class DataLoader {
                 );
 
                 // Extract the new data to set
-                const newData = foundData.map(
-                  (data) => data.context[idMapping.targetField]
-                );
+                const newData = foundData.map((data) => {
+                  const valueFound = this.getValueFromData(
+                    data.finalData,
+                    data.context,
+                    idMapping.targetField
+                  );
+                  return valueFound;
+                });
 
                 // Handle cases where current data is an array
                 if (isFieldToSetArray) {
@@ -587,19 +616,32 @@ export class DataLoader {
                     collectionData.data[i].finalData[fieldToSetKey] =
                       Array.isArray(newData) ? newData : [newData];
                   } else {
-                    // Merge arrays if new data is non-empty array and filter for uniqueness
-                    collectionData.data[i].finalData[fieldToSetKey] = [
-                      ...new Set(
-                        [
-                          ...(Array.isArray(currentDataFiltered)
-                            ? currentDataFiltered
-                            : [currentDataFiltered]),
-                          ...newData,
-                        ].filter(
-                          (value: any) => `${value}` !== `${valueToMatch}`
-                        )
-                      ),
-                    ];
+                    if (Array.isArray(currentDataFiltered)) {
+                      // Convert current data to array and merge if new data is non-empty array
+                      collectionData.data[i].finalData[fieldToSetKey] = [
+                        ...new Set(
+                          [...currentDataFiltered, ...newData].filter(
+                            (value: any) =>
+                              `${value}` !== `${valueToMatch}` && value
+                          )
+                        ),
+                      ];
+                    } else {
+                      // Merge arrays if new data is non-empty array and filter for uniqueness
+                      collectionData.data[i].finalData[fieldToSetKey] = [
+                        ...new Set(
+                          [
+                            ...(Array.isArray(currentDataFiltered)
+                              ? currentDataFiltered
+                              : [currentDataFiltered]),
+                            ...newData,
+                          ].filter(
+                            (value: any) =>
+                              `${value}` !== `${valueToMatch}` && value
+                          )
+                        ),
+                      ];
+                    }
                   }
                 } else {
                   if (!currentDataFiltered) {
@@ -612,7 +654,8 @@ export class DataLoader {
                     collectionData.data[i].finalData[fieldToSetKey] = [
                       ...new Set(
                         [currentDataFiltered, ...newData].filter(
-                          (value: any) => `${value}` !== `${valueToMatch}`
+                          (value: any) =>
+                            `${value}` !== `${valueToMatch}` && value
                         )
                       ),
                     ].slice(0, 1)[0];
@@ -635,52 +678,57 @@ export class DataLoader {
   }
 
   private writeMapsToJsonFile() {
-    const outputDir = path.resolve(process.cwd());
-    const outputFile = path.join(outputDir, "dataLoaderOutput.json");
-
-    const dataToWrite = {
-      // Convert Maps to arrays of entries for serialization
-      oldIdToNewIdPerCollectionMap: Array.from(
-        this.oldIdToNewIdPerCollectionMap.entries()
-      ).map(([key, value]) => {
-        return {
-          collection: key,
-          data: Array.from(value.entries()),
-        };
-      }),
-      mergedUserMap: Array.from(this.mergedUserMap.entries()),
-      dataFromCollections: Array.from(this.importMap.entries()).map(
-        ([key, value]) => {
-          return {
-            collection: key,
-            data: value.data.map((item: any) => item.finalData),
-          };
-        }
-      ),
-      // emailToUserIdMap: Array.from(this.emailToUserIdMap.entries()),
-      // phoneToUserIdMap: Array.from(this.phoneToUserIdMap.entries()),
-    };
+    const outputDir = path.resolve(process.cwd(), "zlogs");
 
-    // Use JSON.stringify with a replacer function to handle Maps
-    const replacer = (key: any, value: any) => {
-      if (value instanceof Map) {
-        return Array.from(value.entries());
-      }
-      return value;
-    };
+    // Ensure the logs directory exists
+    if (!fs.existsSync(outputDir)) {
+      fs.mkdirSync(outputDir);
+    }
 
-    fs.writeFile(
-      outputFile,
-      JSON.stringify(dataToWrite, replacer, 2),
-      "utf8",
-      (err) => {
+    // Helper function to write data to a file
+    const writeToFile = (fileName: string, data: any) => {
+      const outputFile = path.join(outputDir, fileName);
+      fs.writeFile(outputFile, JSON.stringify(data, null, 2), "utf8", (err) => {
         if (err) {
-          console.error("Error writing data to JSON file:", err);
+          console.error(`Error writing data to ${fileName}:`, err);
           return;
         }
-        console.log(`Data successfully written to ${outputFile}`);
-      }
+        console.log(`Data successfully written to ${fileName}`);
+      });
+    };
+
+    // Convert Maps to arrays of entries for serialization
+    const oldIdToNewIdPerCollectionMap = Array.from(
+      this.oldIdToNewIdPerCollectionMap.entries()
+    ).map(([key, value]) => {
+      return {
+        collection: key,
+        data: Array.from(value.entries()),
+      };
+    });
+
+    const mergedUserMap = Array.from(this.mergedUserMap.entries());
+
+    // Write each part to a separate file
+    writeToFile(
+      "oldIdToNewIdPerCollectionMap.json",
+      oldIdToNewIdPerCollectionMap
     );
+    writeToFile("mergedUserMap.json", mergedUserMap);
+
+    // Write each collection's data to a separate file
+    this.importMap.forEach((value, key) => {
+      const data = {
+        collection: key,
+        data: value.data.map((item: any) => {
+          return {
+            finalData: item.finalData,
+            context: item.context,
+          };
+        }),
+      };
+      writeToFile(`${key}.json`, data);
+    });
   }
 
   /**
@@ -723,8 +771,14 @@ export class DataLoader {
     // Check for duplicate email and phone
     if (email && this.emailToUserIdMap.has(email)) {
       existingId = this.emailToUserIdMap.get(email);
+      if (phone && !this.phoneToUserIdMap.has(phone)) {
+        this.phoneToUserIdMap.set(phone, newId);
+      }
     } else if (phone && this.phoneToUserIdMap.has(phone)) {
       existingId = this.phoneToUserIdMap.get(phone);
+      if (email && !this.emailToUserIdMap.has(email)) {
+        this.emailToUserIdMap.set(email, newId);
+      }
     } else {
       if (email) this.emailToUserIdMap.set(email, newId);
       if (phone) this.phoneToUserIdMap.set(phone, newId);
@@ -827,9 +881,6 @@ export class DataLoader {
       this.oldIdToNewIdPerCollectionMap
         .set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
         .get(this.getCollectionKey(collection.name));
-    console.log(
-      `${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`
-    );
     if (!operationId) {
       throw new Error(
         `No import operation found for collection ${collection.name}`
@@ -963,7 +1014,10 @@ export class DataLoader {
             currentData.data[i].finalData,
             transformedItem
           );
-          currentData.data[i].context = context;
+          currentData.data[i].context = {
+            ...currentData.data[i].context,
+            ...context,
+          };
           currentData.data[i].importDef = newImportDef;
           this.importMap.set(
             this.getCollectionKey(collection.name),
@@ -1030,9 +1084,6 @@ export class DataLoader {
       this.oldIdToNewIdPerCollectionMap
         .set(this.getCollectionKey(collection.name), oldIdToNewIdMapNew)
         .get(this.getCollectionKey(collection.name));
-    console.log(
-      `${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`
-    );
     const isRegions = collection.name.toLowerCase() === "regions";
     // Iterate through each item in the raw data
     for (const item of rawData) {
@@ -1054,6 +1105,9 @@ export class DataLoader {
         item,
         importDef.attributeMappings
       );
+      if (collection.name.toLowerCase() === "councils") {
+        console.log("Transformed Council: ", transformedData);
+      }
       if (isRegions) {
         logger.info(
           `Transformed region: ${JSON.stringify(transformedData, null, 2)}`
@@ -1167,7 +1221,6 @@ export class DataLoader {
 
       // Try to find itemDataToUpdate using updateMapping
       if (importDef.updateMapping) {
-        console.log(importDef.updateMapping);
         oldId =
           item[importDef.updateMapping.originalIdField] ||
           transformedData[importDef.updateMapping.originalIdField];
@@ -1199,7 +1252,7 @@ export class DataLoader {
         oldId =
           item[importDef.primaryKeyField] ||
           transformedData[importDef.primaryKeyField];
-        if (oldId) {
+        if (oldId && oldId.length > 0) {
           newId = oldIdToNewIdMap?.get(`${oldId}`);
           if (
             !newId &&

package/src/migrations/importController.ts

@@ -26,7 +26,10 @@ import {
   OperationSchema,
 } from "./backup.js";
 import { DataLoader, type CollectionImportData } from "./dataLoader.js";
-import { transferDocumentsBetweenDbsLocalToLocal } from "./collections.js";
+import {
+  documentExists,
+  transferDocumentsBetweenDbsLocalToLocal,
+} from "./collections.js";
 import { transferDatabaseLocalToLocal } from "./databases.js";
 import { transferStorageLocalToLocal } from "./storage.js";
 
@@ -244,13 +247,43 @@ export class ImportController {
       for (let i = 0; i < dataSplit.length; i++) {
         const batches = dataSplit[i];
         console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
-        const batchPromises = batches.map((item) => {
+
+        // const documentExistsPromises = batches.map(async (item) => {
+        //   try {
+        //     const id =
+        //       item.finalData.docId ||
+        //       item.finalData.userId ||
+        //       item.context.docId ||
+        //       item.context.userId;
+
+        //     if (!item.finalData) {
+        //       return Promise.resolve(null);
+        //     }
+        //     return tryAwaitWithRetry(
+        //       async () =>
+        //         await documentExists(
+        //           this.database,
+        //           db.$id,
+        //           collection.$id,
+        //           item.finalData
+        //         )
+        //     );
+        //   } catch (error) {
+        //     console.error(error);
+        //     return Promise.resolve(null);
+        //   }
+        // });
+
+        // const documentExistsResults = await Promise.all(documentExistsPromises);
+
+        const batchPromises = batches.map((item, index) => {
           try {
             const id =
               item.finalData.docId ||
               item.finalData.userId ||
               item.context.docId ||
               item.context.userId;
+
             if (item.finalData.hasOwnProperty("userId")) {
               delete item.finalData.userId;
             }
@@ -274,6 +307,7 @@ export class ImportController {
             return Promise.resolve();
           }
         });
+
         // Wait for all promises in the current batch to resolve
         await Promise.all(batchPromises);
         console.log(`Completed batch ${i + 1} of ${dataSplit.length}`);

package/src/migrations/logging.ts

@@ -1,16 +1,29 @@
 import winston from "winston";
+import fs from "fs";
+import path from "path";
+
+// Ensure the logs directory exists
+const logDir = path.join(process.cwd(), "zlogs");
+if (!fs.existsSync(logDir)) {
+  fs.mkdirSync(logDir);
+}
 
 export const logger = winston.createLogger({
-  level: "info",
+  level: "debug",
   format: winston.format.json({ space: 2 }),
   defaultMeta: { service: "appwrite-utils-cli" },
   transports: [
-    //
-    // - Write all logs with importance level of `error` or less to `error.log`
-    // - Write all logs with importance level of `info` or less to `combined.log`
-    //
-    new winston.transports.File({ filename: "error.log", level: "error" }),
-    new winston.transports.File({ filename: "warn.log", level: "warn" }),
-    new winston.transports.File({ filename: "combined.log" }),
+    new winston.transports.File({
+      filename: path.join(logDir, "error.log"),
+      level: "error",
+    }),
+    new winston.transports.File({
+      filename: path.join(logDir, "warn.log"),
+      level: "warn",
+    }),
+    new winston.transports.File({
+      filename: path.join(logDir, "info.log"),
+      level: "info",
+    }),
   ],
 });