appwrite-utils-cli 0.0.65 → 0.0.67

package/README.md

@@ -132,6 +132,7 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 ### Changelog
 
+- 0.0.67: Fixed `updates` in `importDef`'s update mappings overwriting postImportActions from the original
 - 0.0.57: Fixed `dataLoader`'s `idMapping`'s giving me issues
 - 0.0.55: Added `documentExists` check to batch creation functionality to try to prevent duplicates
 - 0.0.54: Various fixes in here

package/dist/migrations/backup.d.ts

@@ -319,6 +319,7 @@ export declare const getMigrationCollectionSchemas: () => {
                     fieldToSet?: string | undefined;
                     targetFieldToMatch?: string | undefined;
                 }[] | undefined;
+                createUsers?: boolean | null | undefined;
                 updateMapping?: {
                     targetField: string;
                     originalIdField: string;
@@ -573,6 +574,7 @@ export declare const getMigrationCollectionSchemas: () => {
                     fieldToSet?: string | undefined;
                     targetFieldToMatch?: string | undefined;
                 }[] | undefined;
+                createUsers?: boolean | null | undefined;
                 updateMapping?: {
                     targetField: string;
                     originalIdField: string;

package/dist/migrations/dataLoader.d.ts

@@ -380,6 +380,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }>, "many">>;
+            createUsers: z.ZodOptional<z.ZodNullable<z.ZodDefault<z.ZodBoolean>>>;
             updateMapping: z.ZodOptional<z.ZodObject<{
                 originalIdField: z.ZodString;
                 targetField: z.ZodString;
@@ -494,6 +495,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -529,6 +531,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -623,16 +626,6 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             relatedCollection: string;
             relationType: "oneToMany" | "manyToOne" | "oneToOne" | "manyToMany";
             twoWay: boolean;
-            /**
-             * Generates attribute mappings with post-import actions based on the provided attribute mappings.
-             * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
-             * and update the field with the file's ID if necessary.
-             *
-             * @param attributeMappings - The attribute mappings from the import definition.
-             * @param context - The context object containing information about the database, collection, and document.
-             * @param item - The item being imported, used for resolving template paths in fileData mappings.
-             * @returns The attribute mappings updated with any necessary post-import actions.
-             */
             twoWayKey: string;
             onDelete: "setNull" | "cascade" | "restrict";
             side: "parent" | "child";
@@ -689,6 +682,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -845,6 +839,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -880,6 +875,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }>, "many">>;
+            createUsers: z.ZodOptional<z.ZodNullable<z.ZodDefault<z.ZodBoolean>>>;
             updateMapping: z.ZodOptional<z.ZodObject<{
                 originalIdField: z.ZodString;
                 targetField: z.ZodString;
@@ -994,6 +990,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -1029,6 +1026,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -1069,6 +1067,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -1109,6 +1108,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -1151,6 +1151,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -1245,16 +1246,6 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             relatedCollection: string;
             relationType: "oneToMany" | "manyToOne" | "oneToOne" | "manyToMany";
             twoWay: boolean;
-            /**
-             * Generates attribute mappings with post-import actions based on the provided attribute mappings.
-             * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
-             * and update the field with the file's ID if necessary.
-             *
-             * @param attributeMappings - The attribute mappings from the import definition.
-             * @param context - The context object containing information about the database, collection, and document.
-             * @param item - The item being imported, used for resolving template paths in fileData mappings.
-             * @returns The attribute mappings updated with any necessary post-import actions.
-             */
             twoWayKey: string;
             onDelete: "setNull" | "cascade" | "restrict";
             side: "parent" | "child";
@@ -1311,6 +1302,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -1357,6 +1349,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -1510,6 +1503,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 fieldToSet?: string | undefined;
                 targetFieldToMatch?: string | undefined;
             }[] | undefined;
+            createUsers?: boolean | null | undefined;
             updateMapping?: {
                 targetField: string;
                 originalIdField: string;
@@ -1561,6 +1555,7 @@ export declare class DataLoader {
                     fieldToSet?: string | undefined;
                     targetFieldToMatch?: string | undefined;
                 }[] | undefined;
+                createUsers?: boolean | null | undefined;
                 updateMapping?: {
                     targetField: string;
                     originalIdField: string;
@@ -1655,16 +1650,6 @@ export declare class DataLoader {
                 relatedCollection: string;
                 relationType: "oneToMany" | "manyToOne" | "oneToOne" | "manyToMany";
                 twoWay: boolean;
-                /**
-                 * Generates attribute mappings with post-import actions based on the provided attribute mappings.
-                 * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
-                 * and update the field with the file's ID if necessary.
-                 *
-                 * @param attributeMappings - The attribute mappings from the import definition.
-                 * @param context - The context object containing information about the database, collection, and document.
-                 * @param item - The item being imported, used for resolving template paths in fileData mappings.
-                 * @returns The attribute mappings updated with any necessary post-import actions.
-                 */
                 twoWayKey: string;
                 onDelete: "setNull" | "cascade" | "restrict";
                 side: "parent" | "child";
@@ -1721,6 +1706,7 @@ export declare class DataLoader {
                     fieldToSet?: string | undefined;
                     targetFieldToMatch?: string | undefined;
                 }[] | undefined;
+                createUsers?: boolean | null | undefined;
                 updateMapping?: {
                     targetField: string;
                     originalIdField: string;
@@ -1737,6 +1723,7 @@ export declare class DataLoader {
     private mergedUserMap;
     private emailToUserIdMap;
     private phoneToUserIdMap;
+    private userIdSet;
     userExistsMap: Map<string, boolean>;
     private shouldWriteFile;
     constructor(appwriteFolderPath: string, importDataActions: ImportDataActions, database: Databases, config: AppwriteConfig, shouldWriteFile?: boolean);

package/dist/migrations/dataLoader.js

@@ -45,6 +45,7 @@ export class DataLoader {
     // Maps to hold email and phone to user ID mappings for unique-ness in User Accounts
     emailToUserIdMap = new Map();
     phoneToUserIdMap = new Map();
+    userIdSet = new Set();
     userExistsMap = new Map();
     shouldWriteFile = false;
     // Constructor to initialize the DataLoader with necessary configurations
@@ -173,6 +174,7 @@ export class DataLoader {
         let newId = ID.unique();
         let condition = this.checkMapValuesForId(newId, collectionName) ||
             this.userExistsMap.has(newId) ||
+            this.userIdSet.has(newId) ||
             this.importMap
                 .get(this.getCollectionKey("users"))
                 ?.data.some((user) => user.finalData.docId === newId || user.finalData.userId === newId);
@@ -181,6 +183,7 @@ export class DataLoader {
             condition =
                 this.checkMapValuesForId(newId, collectionName) ||
                     this.userExistsMap.has(newId) ||
+                    this.userIdSet.has(newId) ||
                     this.importMap
                         .get(this.getCollectionKey("users"))
                         ?.data.some((user) => user.finalData.docId === newId || user.finalData.userId === newId);
@@ -260,12 +263,13 @@ export class DataLoader {
         // Iterate over the users and setup our maps ahead of time for email and phone
         for (const user of allUsers) {
             if (user.email) {
-                this.emailToUserIdMap.set(user.email, user.$id);
+                this.emailToUserIdMap.set(user.email.toLowerCase(), user.$id);
             }
             if (user.phone) {
                 this.phoneToUserIdMap.set(user.phone, user.$id);
             }
             this.userExistsMap.set(user.$id, true);
+            this.userIdSet.add(user.$id);
             let importData = this.importMap.get(this.getCollectionKey("users"));
             if (!importData) {
                 importData = {
@@ -275,11 +279,13 @@ export class DataLoader {
             importData.data.push({
                 finalData: {
                     ...user,
+                    email: user.email?.toLowerCase(),
                     userId: user.$id,
                     docId: user.$id,
                 },
                 context: {
                     ...user,
+                    email: user.email?.toLowerCase(),
                     userId: user.$id,
                     docId: user.$id,
                 },
@@ -320,7 +326,7 @@ export class DataLoader {
                 const createDefs = collection.importDefs.filter((def) => def.type === "create" || !def.type);
                 const updateDefs = collection.importDefs.filter((def) => def.type === "update");
                 for (const createDef of createDefs) {
-                    if (!isUsersCollection) {
+                    if (!isUsersCollection || !createDef.createUsers) {
                         await this.prepareCreateData(db, collection, createDef);
                     }
                     else {
@@ -464,7 +470,9 @@ export class DataLoader {
                                 const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
                                 // Log and skip if no matching data found
                                 if (!foundData.length) {
-                                    console.log(`No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`);
+                                    // console.log(
+                                    //   `No data found for collection ${collectionConfig.name}: - Target collection: ${targetCollectionKey} - Value to match: ${valueToMatch} - Field to set: ${fieldToSetKey} - Target field to match: ${targetFieldKey} - Target field value: ${idMapping.targetField}`
+                                    // );
                                     logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
                                     continue;
                                 }
@@ -581,14 +589,18 @@ export class DataLoader {
      * @returns The transformed item with user-specific keys removed.
      */
     prepareUserData(item, attributeMappings, primaryKeyField, newId) {
-        if (this.userExistsMap.has(newId) ||
+        if (this.userIdSet.has(newId) ||
+            this.userExistsMap.has(newId) ||
             Array.from(this.emailToUserIdMap.values()).includes(newId) ||
             Array.from(this.phoneToUserIdMap.values()).includes(newId)) {
             newId = this.getTrueUniqueId(this.getCollectionKey("users"));
         }
         let transformedItem = this.transformData(item, attributeMappings);
-        const userData = AuthUserCreateSchema.safeParse(transformedItem);
-        if (!userData.success || !(userData.data.email && userData.data.phone)) {
+        let userData = AuthUserCreateSchema.safeParse(transformedItem);
+        if (userData.data?.email) {
+            userData.data.email = userData.data.email.toLowerCase();
+        }
+        if (!userData.success || !(userData.data.email || userData.data.phone)) {
             logger.error(`Invalid user data: ${JSON.stringify(userData.error?.errors, undefined, 2)} or missing email/phone`);
             const userKeys = ["email", "phone", "name", "labels", "prefs"];
             userKeys.forEach((key) => {
@@ -605,7 +617,7 @@ export class DataLoader {
                 },
             };
         }
-        const email = userData.data.email;
+        const email = userData.data.email?.toLowerCase();
         const phone = userData.data.phone;
         let existingId;
         // Check for duplicate email and phone
@@ -653,7 +665,12 @@ export class DataLoader {
             if (userFound) {
                 userFound.finalData.userId = existingId;
                 userFound.finalData.docId = existingId;
-                this.userExistsMap.set(existingId, true);
+                this.userIdSet.add(existingId);
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
             }
             const userKeys = ["email", "phone", "name", "labels", "prefs"];
             userKeys.forEach((key) => {
@@ -688,6 +705,7 @@ export class DataLoader {
         this.importMap.set(this.getCollectionKey("users"), {
             data: [...(usersMap?.data || []), userDataToAdd],
         });
+        this.userIdSet.add(existingId);
         return {
             transformedItem,
             existingId,
@@ -962,17 +980,14 @@ export class DataLoader {
                     item[importDef.updateMapping.originalIdField] ||
                         transformedData[importDef.updateMapping.originalIdField];
                 if (oldId) {
-                    itemDataToUpdate = currentData?.data.find(({ context, rawData, finalData }) => {
+                    itemDataToUpdate = currentData?.data.find(({ context, finalData }) => {
                         const targetField = importDef.updateMapping.targetField ??
                             importDef.updateMapping.originalIdField;
                         return (`${context[targetField]}` === `${oldId}` ||
-                            `${rawData[targetField]}` === `${oldId}` ||
                             `${finalData[targetField]}` === `${oldId}`);
                     });
                     if (itemDataToUpdate) {
-                        newId =
-                            itemDataToUpdate.finalData.docId ||
-                                itemDataToUpdate.context.docId;
+                        newId = itemDataToUpdate.context.docId;
                     }
                 }
             }
@@ -1020,10 +1035,10 @@ export class DataLoader {
             }
             transformedData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
             // Create a context object for the item, including the new ID and transformed data
-            let context = this.createContext(db, collection, item, newId);
+            let context = itemDataToUpdate.context;
             context = this.mergeObjects(context, transformedData);
             // Validate the item before proceeding
-            const isValid = await this.importDataActions.validateItem(item, importDef.attributeMappings, context);
+            const isValid = this.importDataActions.validateItem(item, importDef.attributeMappings, context);
             if (!isValid) {
                 logger.info(`Skipping item: ${JSON.stringify(item, null, 2)} because it's invalid`);
                 continue;
@@ -1038,7 +1053,21 @@ export class DataLoader {
             if (itemDataToUpdate) {
                 itemDataToUpdate.finalData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
                 itemDataToUpdate.context = context;
-                itemDataToUpdate.importDef = newImportDef;
+                // Merge existing importDef with new importDef, focusing only on postImportActions
+                itemDataToUpdate.importDef = {
+                    ...itemDataToUpdate.importDef,
+                    attributeMappings: itemDataToUpdate.importDef?.attributeMappings.map((attrMapping, index) => ({
+                        ...attrMapping,
+                        postImportActions: [
+                            ...(attrMapping.postImportActions || []),
+                            ...(newImportDef.attributeMappings[index]
+                                ?.postImportActions || []),
+                        ],
+                    })) || [],
+                };
+                if (collection.name.toLowerCase() === "councils") {
+                    console.log(`Mappings in update councils: ${JSON.stringify(itemDataToUpdate.importDef.attributeMappings, null, 2)}`);
+                }
             }
             else {
                 currentData.data.push({

package/dist/migrations/users.js

@@ -58,10 +58,10 @@ export class UsersController {
                     Query.cursorAfter(lastDocumentId),
                 ]));
                 allUsers.push(...moreUsers.users);
-                lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
                 if (moreUsers.users.length < 200) {
                     break;
                 }
+                lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
             }
         }
         else {
@@ -99,9 +99,9 @@ export class UsersController {
                 }
             }
             if (e instanceof Error) {
-                logger.error("FAILED CREATING USER: ", e.message);
+                logger.error("FAILED CREATING USER: ", e.message, item);
             }
-            console.log("FAILED CREATING USER: ", e);
+            console.log("FAILED CREATING USER: ", e, item);
             throw e;
         }
     }

package/package.json

@@ -1,54 +1,55 @@
-{
-  "name": "appwrite-utils-cli",
-  "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.65",
-  "main": "src/main.ts",
-  "type": "module",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/zachhandley/AppwriteUtils"
-  },
-  "author": "Zach Handley <zach@blackleafdigital.com> (https://zachhandley.com)",
-  "keywords": [
-    "appwrite",
-    "cli",
-    "utils",
-    "migrations",
-    "data",
-    "database",
-    "import",
-    "migration",
-    "utility"
-  ],
-  "bin": {
-    "appwrite-init": "./dist/init.js",
-    "appwrite-migrate": "./dist/main.js"
-  },
-  "scripts": {
-    "build": "bun run tsc",
-    "init": "tsx --no-cache src/init.ts",
-    "migrate": "tsx --no-cache src/main.ts",
-    "deploy": "bun run build && npm publish --access public",
-    "postinstall": "echo 'This package is intended for CLI use only and should not be added as a dependency in other projects.'"
-  },
-  "dependencies": {
-    "@types/inquirer": "^9.0.7",
-    "appwrite-utils": "^0.3.2",
-    "commander": "^12.0.0",
-    "inquirer": "^9.2.20",
-    "js-yaml": "^4.1.0",
-    "lodash": "^4.17.21",
-    "luxon": "^3.4.4",
-    "nanostores": "^0.10.3",
-    "node-appwrite": "^12.0.1",
-    "tsx": "^4.9.3",
-    "winston": "^3.13.0",
-    "zod": "^3.22.4"
-  },
-  "devDependencies": {
-    "@types/js-yaml": "^4.0.9",
-    "@types/lodash": "^4.17.0",
-    "@types/luxon": "^3.4.2",
-    "typescript": "^5.0.0"
-  }
-}
+{
+  "name": "appwrite-utils-cli",
+  "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
+  "version": "0.0.67",
+  "main": "src/main.ts",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/zachhandley/AppwriteUtils"
+  },
+  "author": "Zach Handley <zach@blackleafdigital.com> (https://zachhandley.com)",
+  "keywords": [
+    "appwrite",
+    "cli",
+    "utils",
+    "migrations",
+    "data",
+    "database",
+    "import",
+    "migration",
+    "utility"
+  ],
+  "bin": {
+    "appwrite-init": "./dist/init.js",
+    "appwrite-migrate": "./dist/main.js"
+  },
+  "scripts": {
+    "build": "bun run tsc",
+    "init": "tsx --no-cache src/init.ts",
+    "migrate": "tsx --no-cache src/main.ts",
+    "deploy": "bun run build && npm publish --access public",
+    "postinstall": "echo 'This package is intended for CLI use only and should not be added as a dependency in other projects.'"
+  },
+  "dependencies": {
+    "@types/inquirer": "^9.0.7",
+    "appwrite-utils": "^0.3.3",
+    "commander": "^12.0.0",
+    "inquirer": "^9.2.20",
+    "js-yaml": "^4.1.0",
+    "lodash": "^4.17.21",
+    "luxon": "^3.4.4",
+    "nanostores": "^0.10.3",
+    "node-appwrite": "^12.0.1",
+    "tsx": "^4.9.3",
+    "ulid": "^2.3.0",
+    "winston": "^3.13.0",
+    "zod": "^3.22.4"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/lodash": "^4.17.0",
+    "@types/luxon": "^3.4.2",
+    "typescript": "^5.0.0"
+  }
+}

package/src/migrations/dataLoader.ts

@@ -64,6 +64,7 @@ export class DataLoader {
   // Maps to hold email and phone to user ID mappings for unique-ness in User Accounts
   private emailToUserIdMap = new Map<string, string>();
   private phoneToUserIdMap = new Map<string, string>();
+  private userIdSet = new Set<string>();
   userExistsMap = new Map<string, boolean>();
   private shouldWriteFile = false;
 
@@ -212,6 +213,7 @@ export class DataLoader {
     let condition =
       this.checkMapValuesForId(newId, collectionName) ||
       this.userExistsMap.has(newId) ||
+      this.userIdSet.has(newId) ||
       this.importMap
         .get(this.getCollectionKey("users"))
         ?.data.some(
@@ -223,6 +225,7 @@ export class DataLoader {
       condition =
         this.checkMapValuesForId(newId, collectionName) ||
         this.userExistsMap.has(newId) ||
+        this.userIdSet.has(newId) ||
         this.importMap
           .get(this.getCollectionKey("users"))
           ?.data.some(
@@ -330,12 +333,13 @@ export class DataLoader {
     // Iterate over the users and setup our maps ahead of time for email and phone
     for (const user of allUsers) {
       if (user.email) {
-        this.emailToUserIdMap.set(user.email, user.$id);
+        this.emailToUserIdMap.set(user.email.toLowerCase(), user.$id);
       }
       if (user.phone) {
         this.phoneToUserIdMap.set(user.phone, user.$id);
       }
       this.userExistsMap.set(user.$id, true);
+      this.userIdSet.add(user.$id);
       let importData = this.importMap.get(this.getCollectionKey("users"));
       if (!importData) {
         importData = {
@@ -345,11 +349,13 @@ export class DataLoader {
       importData.data.push({
         finalData: {
           ...user,
+          email: user.email?.toLowerCase(),
           userId: user.$id,
           docId: user.$id,
         },
         context: {
           ...user,
+          email: user.email?.toLowerCase(),
           userId: user.$id,
           docId: user.$id,
         },
@@ -398,7 +404,7 @@ export class DataLoader {
           (def: ImportDef) => def.type === "update"
         );
         for (const createDef of createDefs) {
-          if (!isUsersCollection) {
+          if (!isUsersCollection || !createDef.createUsers) {
             await this.prepareCreateData(db, collection, createDef);
           } else {
             // Special handling for users collection if needed
@@ -592,9 +598,9 @@ export class DataLoader {
                 );
                 // Log and skip if no matching data found
                 if (!foundData.length) {
-                  console.log(
-                    `No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`
-                  );
+                  // console.log(
+                  //   `No data found for collection ${collectionConfig.name}: - Target collection: ${targetCollectionKey} - Value to match: ${valueToMatch} - Field to set: ${fieldToSetKey} - Target field to match: ${targetFieldKey} - Target field value: ${idMapping.targetField}`
+                  // );
                   logger.info(
                     `No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(
                       idMapping,
@@ -761,6 +767,7 @@ export class DataLoader {
     };
   } {
     if (
+      this.userIdSet.has(newId) ||
       this.userExistsMap.has(newId) ||
       Array.from(this.emailToUserIdMap.values()).includes(newId) ||
       Array.from(this.phoneToUserIdMap.values()).includes(newId)
@@ -768,8 +775,11 @@ export class DataLoader {
       newId = this.getTrueUniqueId(this.getCollectionKey("users"));
     }
     let transformedItem = this.transformData(item, attributeMappings);
-    const userData = AuthUserCreateSchema.safeParse(transformedItem);
-    if (!userData.success || !(userData.data.email && userData.data.phone)) {
+    let userData = AuthUserCreateSchema.safeParse(transformedItem);
+    if (userData.data?.email) {
+      userData.data.email = userData.data.email.toLowerCase();
+    }
+    if (!userData.success || !(userData.data.email || userData.data.phone)) {
       logger.error(
         `Invalid user data: ${JSON.stringify(
           userData.error?.errors,
@@ -793,7 +803,7 @@ export class DataLoader {
         },
       };
     }
-    const email = userData.data.email;
+    const email = userData.data.email?.toLowerCase();
     const phone = userData.data.phone;
     let existingId: string | undefined;
 
@@ -836,7 +846,12 @@ export class DataLoader {
       if (userFound) {
         userFound.finalData.userId = existingId;
         userFound.finalData.docId = existingId;
-        this.userExistsMap.set(existingId, true);
+        this.userIdSet.add(existingId);
+        transformedItem = {
+          ...transformedItem,
+          userId: existingId,
+          docId: existingId,
+        };
       }
 
       const userKeys = ["email", "phone", "name", "labels", "prefs"];
@@ -873,6 +888,7 @@ export class DataLoader {
     this.importMap.set(this.getCollectionKey("users"), {
       data: [...(usersMap?.data || []), userDataToAdd],
     });
+    this.userIdSet.add(existingId);
 
     return {
       transformedItem,
@@ -1281,23 +1297,20 @@ export class DataLoader {
           transformedData[importDef.updateMapping.originalIdField];
         if (oldId) {
           itemDataToUpdate = currentData?.data.find(
-            ({ context, rawData, finalData }) => {
+            ({ context, finalData }) => {
               const targetField =
                 importDef.updateMapping!.targetField ??
                 importDef.updateMapping!.originalIdField;
 
               return (
                 `${context[targetField]}` === `${oldId}` ||
-                `${rawData[targetField]}` === `${oldId}` ||
                 `${finalData[targetField]}` === `${oldId}`
               );
             }
           );
 
           if (itemDataToUpdate) {
-            newId =
-              itemDataToUpdate.finalData.docId ||
-              itemDataToUpdate.context.docId;
+            newId = itemDataToUpdate.context.docId;
           }
         }
       }
@@ -1391,11 +1404,11 @@ export class DataLoader {
       );
 
       // Create a context object for the item, including the new ID and transformed data
-      let context = this.createContext(db, collection, item, newId);
+      let context = itemDataToUpdate.context;
       context = this.mergeObjects(context, transformedData);
 
       // Validate the item before proceeding
-      const isValid = await this.importDataActions.validateItem(
+      const isValid = this.importDataActions.validateItem(
         item,
         importDef.attributeMappings,
         context
@@ -1427,7 +1440,31 @@ export class DataLoader {
           transformedData
         );
         itemDataToUpdate.context = context;
-        itemDataToUpdate.importDef = newImportDef;
+        // Merge existing importDef with new importDef, focusing only on postImportActions
+        itemDataToUpdate.importDef = {
+          ...itemDataToUpdate.importDef,
+          attributeMappings:
+            itemDataToUpdate.importDef?.attributeMappings.map(
+              (attrMapping, index) => ({
+                ...attrMapping,
+                postImportActions: [
+                  ...(attrMapping.postImportActions || []),
+                  ...(newImportDef.attributeMappings[index]
+                    ?.postImportActions || []),
+                ],
+              })
+            ) || [],
+        } as ImportDef;
+
+        if (collection.name.toLowerCase() === "councils") {
+          console.log(
+            `Mappings in update councils: ${JSON.stringify(
+              itemDataToUpdate.importDef.attributeMappings,
+              null,
+              2
+            )}`
+          );
+        }
       } else {
         currentData!.data.push({
           rawData: item,

package/src/migrations/users.ts

@@ -89,10 +89,10 @@ export class UsersController {
             ])
         );
         allUsers.push(...moreUsers.users);
-        lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
         if (moreUsers.users.length < 200) {
           break;
         }
+        lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
       }
     } else {
       allUsers.push(...users.users);
@@ -141,9 +141,9 @@ export class UsersController {
         }
       }
       if (e instanceof Error) {
-        logger.error("FAILED CREATING USER: ", e.message);
+        logger.error("FAILED CREATING USER: ", e.message, item);
       }
-      console.log("FAILED CREATING USER: ", e);
+      console.log("FAILED CREATING USER: ", e, item);
       throw e;
     }
   }