appwrite-utils-cli 0.0.260 → 0.0.262

package/dist/migrations/converters.js

@@ -10,7 +10,7 @@ export const converterFunctions = {
     anyToString(value) {
         if (value == null)
             return null;
-        return typeof value === "string" ? value : JSON.stringify(value);
+        return typeof value === "string" ? value : `${value}`;
     },
     /**
      * Converts any value to a number. Returns null for non-numeric values, null, or undefined.

package/dist/migrations/dataLoader.d.ts

@@ -1,5 +1,5 @@
 import type { ImportDataActions } from "./importDataActions.js";
-import { type AppwriteConfig, type AttributeMappings, type ConfigCollection, type ConfigDatabase, type ImportDef } from "./schema.js";
+import { type AppwriteConfig, type AttributeMappings, type ConfigCollection, type ConfigDatabase, type IdMapping, type ImportDef } from "./schema.js";
 import { z } from "zod";
 import { type Databases } from "node-appwrite";
 export declare const CollectionImportDataSchema: z.ZodObject<{
@@ -232,7 +232,14 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             elements?: string[] | undefined;
             xdefault?: string | null | undefined;
         }>]>, z.ZodObject<{
-            key: z.ZodString;
+            key: z.ZodString; /**
+             * Transforms the given item based on the provided attribute mappings.
+             * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
+             *
+             * @param item - The item to be transformed.
+             * @param attributeMappings - The mappings that define how each attribute should be transformed.
+             * @returns The transformed item.
+             */
             type: z.ZodDefault<z.ZodLiteral<"relationship">>;
             error: z.ZodDefault<z.ZodString>;
             required: z.ZodDefault<z.ZodBoolean>;
@@ -1572,7 +1579,7 @@ export declare class DataLoader {
      * @param target - The target object with values to update the source object.
      * @returns The updated source object.
      */
-    mergeObjects(source: Record<string, any>, target: Record<string, any>): Record<string, any>;
+    mergeObjects(source: any, update: any): any;
     loadData(importDef: ImportDef): Promise<any[]>;
     checkMapValuesForId(newId: string, collectionName: string): string | false;
     getTrueUniqueId(collectionName: string): string;
@@ -1590,6 +1597,7 @@ export declare class DataLoader {
     getAllUsers(): Promise<import("node-appwrite").Models.User<import("node-appwrite").Models.Preferences>[]>;
     start(dbId: string): Promise<void>;
     updateReferencesInRelatedCollections(): Promise<void>;
+    findNewIdForOldId(oldId: string, idMapping: IdMapping): string | undefined;
     private writeMapsToJsonFile;
     /**
      * Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,

package/dist/migrations/dataLoader.js

@@ -10,6 +10,7 @@ import { findOrCreateOperation, updateOperation } from "./migrationHelper.js";
 import { AuthUserCreateSchema } from "../schemas/authUser.js";
 import _ from "lodash";
 import { UsersController } from "./users.js";
+import { finalizeByAttributeMap } from "../utils/helperFunctions.js";
 // Define a schema for the structure of collection import data using Zod for validation
 export const CollectionImportDataSchema = z.object({
     // Optional collection creation schema
@@ -69,17 +70,33 @@ export class DataLoader {
      * @param target - The target object with values to update the source object.
      * @returns The updated source object.
      */
-    mergeObjects(source, target) {
-        Object.keys(target).forEach((key) => {
-            const targetValue = target[key];
-            if (source.hasOwnProperty(key) &&
-                targetValue !== null &&
-                targetValue !== undefined &&
-                targetValue !== "") {
-                source[key] = targetValue;
+    mergeObjects(source, update) {
+        // Create a new object to hold the merged result
+        const result = { ...source };
+        Object.keys(update).forEach((key) => {
+            const sourceValue = source[key];
+            const updateValue = update[key];
+            // If the update value is an array, concatenate and remove duplicates
+            if (Array.isArray(updateValue)) {
+                const sourceArray = Array.isArray(sourceValue) ? sourceValue : [];
+                result[key] = [...new Set([...sourceArray, ...updateValue])];
+            }
+            // If the update value is an object, recursively merge
+            else if (updateValue !== null &&
+                typeof updateValue === "object" &&
+                !(updateValue instanceof Date)) {
+                result[key] = this.mergeObjects(sourceValue, updateValue);
+            }
+            // If the update value is not nullish, overwrite the source value
+            else if (updateValue !== null && updateValue !== undefined) {
+                result[key] = updateValue;
+            }
+            // If the update value is nullish, keep the original value unless it doesn't exist
+            else if (sourceValue === undefined) {
+                result[key] = updateValue;
             }
         });
-        return source;
+        return result;
     }
     // Method to load data from a file specified in the import definition
     async loadData(importDef) {
@@ -255,6 +272,7 @@ export class DataLoader {
             const collectionData = this.importMap.get(collectionKey);
             if (!collectionData || !collectionData.data)
                 continue;
+            console.log(`Updating references for collection: ${collectionConfig.name}`);
             // Iterate over each data item in the current collection
             for (const item of collectionData.data) {
                 let needsUpdate = false;
@@ -264,91 +282,62 @@ export class DataLoader {
                         if (importDef.idMappings) {
                             // Iterate over each idMapping defined for the current import definition
                             for (const idMapping of importDef.idMappings) {
-                                const oldId = item.context[idMapping.sourceField];
-                                if (oldId) {
+                                const oldIds = Array.isArray(item.context[idMapping.sourceField])
+                                    ? item.context[idMapping.sourceField]
+                                    : [item.context[idMapping.sourceField]];
+                                oldIds.forEach((oldId) => {
                                     let newIdForOldId;
-                                    // Your existing logic to resolve newIdForOldId here...
+                                    // Handling users merged into a new ID
+                                    newIdForOldId = this.findNewIdForOldId(oldId, idMapping);
                                     if (newIdForOldId) {
-                                        // Determine the field to update, handling arrays appropriately
-                                        const fieldToUpdate = idMapping.fieldToSet || idMapping.targetField;
-                                        if (Array.isArray(item.finalData[fieldToUpdate])) {
-                                            // If the target field is an array, append the new ID
-                                            item.finalData[fieldToUpdate].push(newIdForOldId);
+                                        const targetField = idMapping.fieldToSet || idMapping.targetField;
+                                        const isArray = collectionConfig.attributes.some((attribute) => attribute.key === targetField && attribute.array);
+                                        // Properly update the target field based on whether it should be an array
+                                        if (isArray) {
+                                            if (!Array.isArray(item.finalData[targetField])) {
+                                                item.finalData[targetField] = [newIdForOldId];
+                                            }
+                                            else if (!item.finalData[targetField].includes(newIdForOldId)) {
+                                                item.finalData[targetField].push(newIdForOldId);
+                                            }
                                         }
                                         else {
-                                            // Otherwise, directly set the new ID
-                                            item.finalData[fieldToUpdate] = newIdForOldId;
+                                            item.finalData[targetField] = newIdForOldId;
                                         }
-                                        console.log(`Updated ${oldId} to ${newIdForOldId}`);
                                         needsUpdate = true;
                                     }
-                                }
+                                });
                             }
                         }
                     }
                 }
+                // Update the importMap if changes were made to the item
                 if (needsUpdate) {
-                    // Re-transform the item's finalData using its attribute mappings
-                    const importDef = item.importDef; // Assuming importDef is available in the item
-                    if (importDef && importDef.attributeMappings) {
-                        item.finalData = await this.transformData(item.finalData, importDef.attributeMappings);
-                    }
                     this.importMap.set(collectionKey, collectionData);
+                    logger.info(`Updated item: ${JSON.stringify(item.finalData, undefined, 2)}`);
                 }
             }
         }
     }
-    // async updateReferencesInRelatedCollections() {
-    //   // Process each collection defined in the config
-    //   for (const collection of this.config.collections) {
-    //     // Ensure we handle only those definitions with idMappings and a primaryKeyField defined
-    //     for (const importDef of collection.importDefs) {
-    //       if (!importDef.idMappings || !importDef.primaryKeyField) continue;
-    //       // Load data for this particular import definition
-    //       const collectionData = this.importMap.get(
-    //         this.getCollectionKey(collection.name)
-    //       )?.data;
-    //       if (!collectionData) continue;
-    //       // Iterate over each item in the collection data
-    //       for (const item of collectionData) {
-    //         let needsUpdate = false;
-    //         // Go through each idMapping defined for the collection
-    //         for (const mapping of importDef.idMappings) {
-    //           const oldReferenceId =
-    //             item[mapping.targetField as keyof typeof item]; // Get the current reference ID from the item
-    //           const referenceCollectionMap =
-    //             this.oldIdToNewIdPerCollectionMap.get(
-    //               this.getCollectionKey(mapping.targetCollection)
-    //             );
-    //           if (
-    //             referenceCollectionMap &&
-    //             referenceCollectionMap.has(oldReferenceId)
-    //           ) {
-    //             // Update the target field with the new reference ID from the mapped collection
-    //             item[mapping.sourceField as keyof typeof item] =
-    //               referenceCollectionMap.get(oldReferenceId);
-    //             needsUpdate = true;
-    //             console.log(
-    //               `Updated item with ${mapping.sourceField} = ${JSON.stringify(
-    //                 item,
-    //                 null,
-    //                 undefined
-    //               )}.`
-    //             );
-    //           }
-    //         }
-    //         // Save changes if any reference was updated
-    //         if (needsUpdate) {
-    //           await this.saveUpdatedItem(
-    //             collection.name,
-    //             item,
-    //             importDef.primaryKeyField
-    //           );
-    //         }
-    //       }
-    //     }
-    //   }
-    // }
+    findNewIdForOldId(oldId, idMapping) {
+        // Check merged users first for any corresponding new ID
+        let newIdForOldId;
+        for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
+            if (oldIds.includes(oldId)) {
+                newIdForOldId = newUserId;
+                break;
+            }
+        }
+        // If no new ID found in merged users, check the old-to-new ID map for the target collection
+        if (!newIdForOldId) {
+            const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
+            const targetOldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(targetCollectionKey);
+            if (targetOldIdToNewIdMap && targetOldIdToNewIdMap.has(oldId)) {
+                newIdForOldId = targetOldIdToNewIdMap.get(oldId);
+            }
+        }
+        return newIdForOldId;
+    }
     writeMapsToJsonFile() {
         const outputDir = path.resolve(process.cwd());
         const outputFile = path.join(outputDir, "dataLoaderOutput.json");
@@ -401,7 +390,6 @@ export class DataLoader {
         // Check for duplicate email and add to emailToUserIdMap if not found
         if (email && email.length > 0) {
             if (this.emailToUserIdMap.has(email)) {
-                logger.error(`Duplicate email found or user exists already: ${email}`);
                 existingId = this.emailToUserIdMap.get(email);
             }
             else {
@@ -411,7 +399,6 @@ export class DataLoader {
         // Check for duplicate phone and add to phoneToUserIdMap if not found
         if (phone && phone.length > 0) {
             if (this.phoneToUserIdMap.has(phone)) {
-                logger.error(`Duplicate phone found: ${phone}`);
                 existingId = this.phoneToUserIdMap.get(phone);
             }
             else {
@@ -728,15 +715,23 @@ export class DataLoader {
                 attributeMappings: mappingsWithActions,
             };
             // Add the item with its context and final data to the current collection data
-            if (currentData) {
+            if (itemDataToUpdate) {
+                // Update the existing item's finalData and context in place
+                itemDataToUpdate.finalData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
+                itemDataToUpdate.context = context;
+                itemDataToUpdate.importDef = newImportDef;
+            }
+            else {
+                // If no existing item matches, then add the new item
                 currentData.data.push({
                     rawData: item,
                     context: context,
                     importDef: newImportDef,
                     finalData: transformedData,
                 });
-                this.importMap.set(this.getCollectionKey(collection.name), currentData);
             }
+            // Since we're modifying currentData in place, we ensure no duplicates are added
+            this.importMap.set(this.getCollectionKey(collection.name), currentData);
         }
     }
     updateReferencesBasedOnAttributeMappings() {

package/dist/migrations/importController.js

@@ -159,6 +159,9 @@ export class ImportController {
                     if (item.finalData.hasOwnProperty("docId")) {
                         delete item.finalData.docId;
                     }
+                    if (!item.finalData) {
+                        return Promise.resolve();
+                    }
                     return this.database
                         .createDocument(db.$id, collection.$id, id, item.finalData)
                         .then(() => {

package/dist/migrations/importDataActions.d.ts

@@ -12,7 +12,7 @@ export declare class ImportDataActions {
     private validityRuleDefinitions;
     private afterImportActionsDefinitions;
     constructor(db: Databases, storage: Storage, config: AppwriteConfig, converterDefinitions: ConverterFunctions, validityRuleDefinitions: ValidationRules, afterImportActionsDefinitions: AfterImportActions);
-    runConverterFunctions(item: any, attributeMappings: AttributeMappings): Promise<any>;
+    runConverterFunctions(item: any, attributeMappings: AttributeMappings): any;
     /**
      * Validates a single data item based on defined validation rules.
      * @param item The data item to validate.

package/dist/migrations/importDataActions.js

@@ -18,7 +18,7 @@ export class ImportDataActions {
         this.validityRuleDefinitions = validityRuleDefinitions;
         this.afterImportActionsDefinitions = afterImportActionsDefinitions;
     }
-    async runConverterFunctions(item, attributeMappings) {
+    runConverterFunctions(item, attributeMappings) {
         const conversionSchema = attributeMappings.reduce((schema, mapping) => {
             schema[mapping.targetKey] = (originalValue) => {
                 return mapping.converters.reduce((value, converterName) => {

package/dist/migrations/schema.d.ts

@@ -746,6 +746,22 @@ export declare const AttributeMappingsSchema: z.ZodArray<z.ZodObject<{
         action: string;
     }[] | undefined;
 }>, "many">;
+export declare const idMappingSchema: z.ZodArray<z.ZodObject<{
+    sourceField: z.ZodString;
+    fieldToSet: z.ZodOptional<z.ZodString>;
+    targetField: z.ZodString;
+    targetCollection: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    targetField: string;
+    sourceField: string;
+    targetCollection: string;
+    fieldToSet?: string | undefined;
+}, {
+    targetField: string;
+    sourceField: string;
+    targetCollection: string;
+    fieldToSet?: string | undefined;
+}>, "many">;
 export declare const importDefSchema: z.ZodObject<{
     type: z.ZodOptional<z.ZodDefault<z.ZodEnum<["create", "update"]>>>;
     filePath: z.ZodString;
@@ -3692,6 +3708,8 @@ export type ConfigDatabases = AppwriteConfig["databases"];
 export type ConfigDatabase = ConfigDatabases[number];
 export type ImportDefs = z.infer<typeof importDefSchemas>;
 export type ImportDef = z.infer<typeof importDefSchema>;
+export type IdMappings = z.infer<typeof idMappingSchema>;
+export type IdMapping = IdMappings[number];
 export type AttributeMappings = z.infer<typeof AttributeMappingsSchema>;
 export type AttributeMapping = AttributeMappings[number];
 export {};

package/dist/migrations/schema.js

@@ -366,6 +366,19 @@ export const AttributeMappingsSchema = z.array(z.object({
         .describe("The after import actions and parameter placeholders (they'll be replaced with the actual data) to use for the import")
         .default([]),
 }));
+export const idMappingSchema = z.array(z.object({
+    sourceField: z
+        .string()
+        .describe("The key of the data in the import data to match in the current data"),
+    fieldToSet: z
+        .string()
+        .optional()
+        .describe("The field to set in the target collection, if different from sourceField"),
+    targetField: z
+        .string()
+        .describe("The field in the target collection to match with sourceField that will then be updated"),
+    targetCollection: z.string().describe("The collection to search"),
+}));
 export const importDefSchema = z
     .object({
     type: z
@@ -382,20 +395,7 @@ export const importDefSchema = z
         .string()
         .default("id")
         .describe("The field in the import data representing the primary key for this import data (if any)"),
-    idMappings: z
-        .array(z.object({
-        sourceField: z
-            .string()
-            .describe("The key of the data in the import data to match in the current data"),
-        fieldToSet: z
-            .string()
-            .optional()
-            .describe("The field to set in the target collection, if different from sourceField"),
-        targetField: z
-            .string()
-            .describe("The field in the target collection to match with sourceField that will then be updated"),
-        targetCollection: z.string().describe("The collection to search"),
-    }))
+    idMappings: idMappingSchema
         .optional()
         .describe("The id mappings for the attribute to map ID's to"),
     updateMapping: z

package/dist/utils/helperFunctions.d.ts

@@ -1,4 +1,6 @@
 import type { Models } from "node-appwrite";
+import type { CollectionImportData } from "src/migrations/dataLoader.js";
+import type { ConfigCollection } from "src/migrations/schema.js";
 export declare const toPascalCase: (str: string) => string;
 export declare const toCamelCase: (str: string) => string;
 export declare const ensureDirectoryExistence: (filePath: string) => true | undefined;
@@ -32,3 +34,4 @@ export declare const getFileViewUrl: (endpoint: string, projectId: string, bucke
  * @return {string} The complete download URL for the file.
  */
 export declare const getFileDownloadUrl: (endpoint: string, projectId: string, bucketId: string, fileId: string, jwt?: Models.Jwt) => string;
+export declare const finalizeByAttributeMap: (appwriteFolderPath: string, collection: ConfigCollection, item: CollectionImportData["data"][number]) => Promise<any>;

package/dist/utils/helperFunctions.js

@@ -70,3 +70,11 @@ export const getFileViewUrl = (endpoint, projectId, bucketId, fileId, jwt) => {
 export const getFileDownloadUrl = (endpoint, projectId, bucketId, fileId, jwt) => {
     return `${endpoint}/storage/buckets/${bucketId}/files/${fileId}/download?project=${projectId}${jwt ? `&jwt=${jwt.jwt}` : ""}`;
 };
+export const finalizeByAttributeMap = async (appwriteFolderPath, collection, item) => {
+    const schemaFolderPath = path.join(appwriteFolderPath, "schemas");
+    const zodSchema = await import(`${schemaFolderPath}/${toCamelCase(collection.name)}.ts`);
+    return zodSchema.parse({
+        ...item.context,
+        ...item.finalData,
+    });
+};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.260",
+  "version": "0.0.262",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/converters.ts

@@ -1,6 +1,6 @@
 import { DateTime } from "luxon";
 import _ from "lodash";
-import type { AppwriteConfig } from "./schema.js";
+import type { AppwriteConfig, ConfigCollection } from "./schema.js";
 
 const { cloneDeep, isObject } = _;
 
@@ -16,7 +16,7 @@ export const converterFunctions = {
    */
   anyToString(value: any): string | null {
     if (value == null) return null;
-    return typeof value === "string" ? value : JSON.stringify(value);
+    return typeof value === "string" ? value : `${value}`;
   },
 
   /**

package/src/migrations/dataLoader.ts

@@ -7,6 +7,7 @@ import {
   type AttributeMappings,
   type ConfigCollection,
   type ConfigDatabase,
+  type IdMapping,
   type ImportDef,
   type ImportDefs,
   type RelationshipAttribute,
@@ -22,6 +23,7 @@ import { findOrCreateOperation, updateOperation } from "./migrationHelper.js";
 import { AuthUserCreateSchema } from "../schemas/authUser.js";
 import _ from "lodash";
 import { UsersController } from "./users.js";
+import { finalizeByAttributeMap } from "../utils/helperFunctions.js";
 // Define a schema for the structure of collection import data using Zod for validation
 export const CollectionImportDataSchema = z.object({
   // Optional collection creation schema
@@ -96,22 +98,38 @@ export class DataLoader {
    * @param target - The target object with values to update the source object.
    * @returns The updated source object.
    */
-  mergeObjects(
-    source: Record<string, any>,
-    target: Record<string, any>
-  ): Record<string, any> {
-    Object.keys(target).forEach((key) => {
-      const targetValue = target[key];
-      if (
-        source.hasOwnProperty(key) &&
-        targetValue !== null &&
-        targetValue !== undefined &&
-        targetValue !== ""
+  mergeObjects(source: any, update: any): any {
+    // Create a new object to hold the merged result
+    const result = { ...source };
+
+    Object.keys(update).forEach((key) => {
+      const sourceValue = source[key];
+      const updateValue = update[key];
+
+      // If the update value is an array, concatenate and remove duplicates
+      if (Array.isArray(updateValue)) {
+        const sourceArray = Array.isArray(sourceValue) ? sourceValue : [];
+        result[key] = [...new Set([...sourceArray, ...updateValue])];
+      }
+      // If the update value is an object, recursively merge
+      else if (
+        updateValue !== null &&
+        typeof updateValue === "object" &&
+        !(updateValue instanceof Date)
       ) {
-        source[key] = targetValue;
+        result[key] = this.mergeObjects(sourceValue, updateValue);
+      }
+      // If the update value is not nullish, overwrite the source value
+      else if (updateValue !== null && updateValue !== undefined) {
+        result[key] = updateValue;
+      }
+      // If the update value is nullish, keep the original value unless it doesn't exist
+      else if (sourceValue === undefined) {
+        result[key] = updateValue;
       }
     });
-    return source;
+
+    return result;
   }
 
   // Method to load data from a file specified in the import definition
@@ -332,6 +350,10 @@ export class DataLoader {
 
       if (!collectionData || !collectionData.data) continue;
 
+      console.log(
+        `Updating references for collection: ${collectionConfig.name}`
+      );
+
       // Iterate over each data item in the current collection
       for (const item of collectionData.data) {
         let needsUpdate = false;
@@ -342,102 +364,82 @@ export class DataLoader {
             if (importDef.idMappings) {
               // Iterate over each idMapping defined for the current import definition
               for (const idMapping of importDef.idMappings) {
-                const oldId = item.context[idMapping.sourceField];
-                if (oldId) {
-                  let newIdForOldId: string | undefined;
-                  // Your existing logic to resolve newIdForOldId here...
+                const oldIds = Array.isArray(
+                  item.context[idMapping.sourceField]
+                )
+                  ? item.context[idMapping.sourceField]
+                  : [item.context[idMapping.sourceField]];
+
+                oldIds.forEach((oldId: any) => {
+                  let newIdForOldId;
+
+                  // Handling users merged into a new ID
+                  newIdForOldId = this.findNewIdForOldId(oldId, idMapping);
 
                   if (newIdForOldId) {
-                    // Determine the field to update, handling arrays appropriately
-                    const fieldToUpdate =
+                    const targetField =
                       idMapping.fieldToSet || idMapping.targetField;
-                    if (Array.isArray(item.finalData[fieldToUpdate])) {
-                      // If the target field is an array, append the new ID
-                      item.finalData[fieldToUpdate].push(newIdForOldId);
+                    const isArray = collectionConfig.attributes.some(
+                      (attribute) =>
+                        attribute.key === targetField && attribute.array
+                    );
+
+                    // Properly update the target field based on whether it should be an array
+                    if (isArray) {
+                      if (!Array.isArray(item.finalData[targetField])) {
+                        item.finalData[targetField] = [newIdForOldId];
+                      } else if (
+                        !item.finalData[targetField].includes(newIdForOldId)
+                      ) {
+                        item.finalData[targetField].push(newIdForOldId);
+                      }
                     } else {
-                      // Otherwise, directly set the new ID
-                      item.finalData[fieldToUpdate] = newIdForOldId;
+                      item.finalData[targetField] = newIdForOldId;
                     }
-                    console.log(`Updated ${oldId} to ${newIdForOldId}`);
                     needsUpdate = true;
                   }
-                }
+                });
               }
             }
           }
         }
 
+        // Update the importMap if changes were made to the item
         if (needsUpdate) {
-          // Re-transform the item's finalData using its attribute mappings
-          const importDef = item.importDef; // Assuming importDef is available in the item
-          if (importDef && importDef.attributeMappings) {
-            item.finalData = await this.transformData(
-              item.finalData,
-              importDef.attributeMappings
-            );
-          }
           this.importMap.set(collectionKey, collectionData);
+          logger.info(
+            `Updated item: ${JSON.stringify(item.finalData, undefined, 2)}`
+          );
         }
       }
     }
   }
 
-  // async updateReferencesInRelatedCollections() {
-  //   // Process each collection defined in the config
-  //   for (const collection of this.config.collections) {
-  //     // Ensure we handle only those definitions with idMappings and a primaryKeyField defined
-  //     for (const importDef of collection.importDefs) {
-  //       if (!importDef.idMappings || !importDef.primaryKeyField) continue;
-
-  //       // Load data for this particular import definition
-  //       const collectionData = this.importMap.get(
-  //         this.getCollectionKey(collection.name)
-  //       )?.data;
-  //       if (!collectionData) continue;
-
-  //       // Iterate over each item in the collection data
-  //       for (const item of collectionData) {
-  //         let needsUpdate = false;
+  findNewIdForOldId(oldId: string, idMapping: IdMapping) {
+    // Check merged users first for any corresponding new ID
+    let newIdForOldId;
+    for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
+      if (oldIds.includes(oldId)) {
+        newIdForOldId = newUserId;
+        break;
+      }
+    }
 
-  //         // Go through each idMapping defined for the collection
-  //         for (const mapping of importDef.idMappings) {
-  //           const oldReferenceId =
-  //             item[mapping.targetField as keyof typeof item]; // Get the current reference ID from the item
-  //           const referenceCollectionMap =
-  //             this.oldIdToNewIdPerCollectionMap.get(
-  //               this.getCollectionKey(mapping.targetCollection)
-  //             );
+    // If no new ID found in merged users, check the old-to-new ID map for the target collection
+    if (!newIdForOldId) {
+      const targetCollectionKey = this.getCollectionKey(
+        idMapping.targetCollection
+      );
+      const targetOldIdToNewIdMap =
+        this.oldIdToNewIdPerCollectionMap.get(targetCollectionKey);
 
-  //           if (
-  //             referenceCollectionMap &&
-  //             referenceCollectionMap.has(oldReferenceId)
-  //           ) {
-  //             // Update the target field with the new reference ID from the mapped collection
-  //             item[mapping.sourceField as keyof typeof item] =
-  //               referenceCollectionMap.get(oldReferenceId);
-  //             needsUpdate = true;
-  //             console.log(
-  //               `Updated item with ${mapping.sourceField} = ${JSON.stringify(
-  //                 item,
-  //                 null,
-  //                 undefined
-  //               )}.`
-  //             );
-  //           }
-  //         }
+      if (targetOldIdToNewIdMap && targetOldIdToNewIdMap.has(oldId)) {
+        newIdForOldId = targetOldIdToNewIdMap.get(oldId);
+      }
+    }
 
-  //         // Save changes if any reference was updated
-  //         if (needsUpdate) {
-  //           await this.saveUpdatedItem(
-  //             collection.name,
-  //             item,
-  //             importDef.primaryKeyField
-  //           );
-  //         }
-  //       }
-  //     }
-  //   }
-  // }
+    return newIdForOldId;
+  }
 
   private writeMapsToJsonFile() {
     const outputDir = path.resolve(process.cwd());
@@ -514,7 +516,6 @@ export class DataLoader {
     // Check for duplicate email and add to emailToUserIdMap if not found
     if (email && email.length > 0) {
       if (this.emailToUserIdMap.has(email)) {
-        logger.error(`Duplicate email found or user exists already: ${email}`);
         existingId = this.emailToUserIdMap.get(email);
       } else {
         this.emailToUserIdMap.set(email, newId);
@@ -524,7 +525,6 @@ export class DataLoader {
     // Check for duplicate phone and add to phoneToUserIdMap if not found
     if (phone && phone.length > 0) {
       if (this.phoneToUserIdMap.has(phone)) {
-        logger.error(`Duplicate phone found: ${phone}`);
         existingId = this.phoneToUserIdMap.get(phone);
       } else {
         this.phoneToUserIdMap.set(phone, newId);
@@ -999,15 +999,25 @@ export class DataLoader {
         attributeMappings: mappingsWithActions,
       };
       // Add the item with its context and final data to the current collection data
-      if (currentData) {
-        currentData.data.push({
+      if (itemDataToUpdate) {
+        // Update the existing item's finalData and context in place
+        itemDataToUpdate.finalData = this.mergeObjects(
+          itemDataToUpdate.finalData,
+          transformedData
+        );
+        itemDataToUpdate.context = context;
+        itemDataToUpdate.importDef = newImportDef;
+      } else {
+        // If no existing item matches, then add the new item
+        currentData!.data.push({
           rawData: item,
           context: context,
           importDef: newImportDef,
           finalData: transformedData,
         });
-        this.importMap.set(this.getCollectionKey(collection.name), currentData);
       }
+      // Since we're modifying currentData in place, we ensure no duplicates are added
+      this.importMap.set(this.getCollectionKey(collection.name), currentData!);
     }
   }
 

package/src/migrations/importController.ts

@@ -236,6 +236,9 @@ export class ImportController {
           if (item.finalData.hasOwnProperty("docId")) {
             delete item.finalData.docId;
           }
+          if (!item.finalData) {
+            return Promise.resolve();
+          }
           return this.database
             .createDocument(db.$id, collection.$id, id, item.finalData)
             .then(() => {

package/src/migrations/importDataActions.ts

@@ -45,7 +45,7 @@ export class ImportDataActions {
     this.afterImportActionsDefinitions = afterImportActionsDefinitions;
   }
 
-  async runConverterFunctions(item: any, attributeMappings: AttributeMappings) {
+  runConverterFunctions(item: any, attributeMappings: AttributeMappings) {
     const conversionSchema = attributeMappings.reduce((schema, mapping) => {
       schema[mapping.targetKey] = (originalValue: any) => {
         return mapping.converters.reduce((value, converterName) => {

package/src/migrations/schema.ts

@@ -435,6 +435,28 @@ export const AttributeMappingsSchema = z.array(
   })
 );
 
+export const idMappingSchema = z.array(
+  z.object({
+    sourceField: z
+      .string()
+      .describe(
+        "The key of the data in the import data to match in the current data"
+      ),
+    fieldToSet: z
+      .string()
+      .optional()
+      .describe(
+        "The field to set in the target collection, if different from sourceField"
+      ),
+    targetField: z
+      .string()
+      .describe(
+        "The field in the target collection to match with sourceField that will then be updated"
+      ),
+    targetCollection: z.string().describe("The collection to search"),
+  })
+);
+
 export const importDefSchema = z
   .object({
     type: z
@@ -457,28 +479,7 @@ export const importDefSchema = z
       .describe(
         "The field in the import data representing the primary key for this import data (if any)"
       ),
-    idMappings: z
-      .array(
-        z.object({
-          sourceField: z
-            .string()
-            .describe(
-              "The key of the data in the import data to match in the current data"
-            ),
-          fieldToSet: z
-            .string()
-            .optional()
-            .describe(
-              "The field to set in the target collection, if different from sourceField"
-            ),
-          targetField: z
-            .string()
-            .describe(
-              "The field in the target collection to match with sourceField that will then be updated"
-            ),
-          targetCollection: z.string().describe("The collection to search"),
-        })
-      )
+    idMappings: idMappingSchema
       .optional()
       .describe("The id mappings for the attribute to map ID's to"),
     updateMapping: z
@@ -622,5 +623,7 @@ export type ConfigDatabases = AppwriteConfig["databases"];
 export type ConfigDatabase = ConfigDatabases[number];
 export type ImportDefs = z.infer<typeof importDefSchemas>;
 export type ImportDef = z.infer<typeof importDefSchema>;
+export type IdMappings = z.infer<typeof idMappingSchema>;
+export type IdMapping = IdMappings[number];
 export type AttributeMappings = z.infer<typeof AttributeMappingsSchema>;
 export type AttributeMapping = AttributeMappings[number];

package/src/utils/helperFunctions.ts

@@ -2,6 +2,8 @@ import type { AppwriteConfig } from "../types.js";
 import type { Models, Storage } from "node-appwrite";
 import fs from "node:fs";
 import path from "node:path";
+import type { CollectionImportData } from "src/migrations/dataLoader.js";
+import type { ConfigCollection } from "src/migrations/schema.js";
 
 export const toPascalCase = (str: string): string => {
   return (
@@ -109,3 +111,18 @@ export const getFileDownloadUrl = (
     jwt ? `&jwt=${jwt.jwt}` : ""
   }`;
 };
+
+export const finalizeByAttributeMap = async (
+  appwriteFolderPath: string,
+  collection: ConfigCollection,
+  item: CollectionImportData["data"][number]
+) => {
+  const schemaFolderPath = path.join(appwriteFolderPath, "schemas");
+  const zodSchema = await import(
+    `${schemaFolderPath}/${toCamelCase(collection.name)}.ts`
+  );
+  return zodSchema.parse({
+    ...item.context,
+    ...item.finalData,
+  });
+};