appwrite-utils-cli 0.0.33 → 0.0.34

package/README.md

@@ -55,17 +55,21 @@ Replace `--args` with the appropriate options:
 - `--staging`: Run tasks in the staging environment.
 - `--dev`: Run tasks in the development environment.
 - `--wipe`: Wipe all databases.
-- `--wipe-docs` or `--wipeDocs`: Wipe all documents in the databases.
+- `--wipe-docs`: Wipe all documents in the databases.
 - `--generate`: Generate TypeScript schemas from database schemas.
 - `--import`: Import data into your databases.
 - `--backup`: Perform a backup of your databases.
-- `--wipe-users` or `--wipeUsers`: Wipe all user data.
-- `--write-data` or `--writeData`: Write converted imported data to file
+- `--wipe-users`: Wipe all user data.
+- `--write-data`: Write converted imported data to file
 - `--sync`: Synchronize your project's config and generate schema for your database
 - `--endpoint`: Set a different endpoint for the migration target
 - `--project`: Set a different project ID for the migration target
 - `--key`: Set a different API key for the migration target
 
+## If you run out of RAM
+
+This happens because Node only allocates 4 GB, it'll only happen if you're importing a ton of data, but you can use `export NODE_OPTIONS="--max-old-space-size=16384"` or the relevant command for your system if that doesn't work, and it'll set the env var to whatever. That's 16 GB, I would recommend `8192` for most.
+
 ### OpenAPI Generation (almost done, in progress)
 
 Recently, I have also added an optional OpenAPI generation for each attribute in the schema. This is because I needed it and because I felt it would be nice to have. This is done using [this package](https://github.com/asteasolutions/zod-to-openapi), many thanks to them.
@@ -82,6 +86,7 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 ### Changelog
 
+- 0.0.34: Fixed the `bin` section of the package.json, apparently you can't use `node` to run it
 - 0.0.33: Fixed `idMappings`, if you are importing data and use the `idMappings` functionality, you can set a `fieldToSet` based on the value of a `sourceField` in the current imported items data (whether it's in the final data or the original), in order to match another field in another collection. So if you had a store, and it had items and the items have a Region ID for instance. You can then, in your regionId of the items, setup an `idMapping` that will allow you to map the value of the `targetField` based on the value of the `targetFieldToMatch` in the `targetCollection`. Sounds complex, but it's very useful. Like psuedo-relationship resolution, without the relationships.
 - 0.0.29: If you use the `description` variable in an attribute and collection, it'll add that description to the generated schemas. This assumes you have `zod-to-openpi`
 - 0.0.275: THINGS ARE NOW IN TYPESCRIPT WOOHOO. No but for reaal, super happy to report that everything has been converted to TypeScript, just way too many changes, I hope you enjoy it!

package/dist/migrations/dataLoader.d.ts

@@ -61,6 +61,14 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             required: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
             array: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
             min: z.ZodOptional<z.ZodNumber>;
+            /**
+             * Prepares the data for creating documents in a collection.
+             * This involves loading the data, transforming it, and handling ID mappings.
+             *
+             * @param db - The database configuration.
+             * @param collection - The collection configuration.
+             * @param importDef - The import definition containing the attribute mappings and other relevant info.
+             */
             max: z.ZodOptional<z.ZodNumber>;
             xdefault: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
             description: z.ZodOptional<z.ZodNullable<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodString>]>>>;
@@ -359,6 +367,16 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             type: z.ZodOptional<z.ZodDefault<z.ZodEnum<["create", "update"]>>>;
             filePath: z.ZodString;
             basePath: z.ZodOptional<z.ZodString>;
+            /**
+             * Generates attribute mappings with post-import actions based on the provided attribute mappings.
+             * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
+             * and update the field with the file's ID if necessary.
+             *
+             * @param attributeMappings - The attribute mappings from the import definition.
+             * @param context - The context object containing information about the database, collection, and document.
+             * @param item - The item being imported, used for resolving template paths in fileData mappings.
+             * @returns The attribute mappings updated with any necessary post-import actions.
+             */
             primaryKeyField: z.ZodDefault<z.ZodString>;
             idMappings: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 sourceField: z.ZodString;

package/dist/migrations/dataLoader.js

@@ -92,7 +92,7 @@ export class DataLoader {
                 result[key] = updateValue;
             }
             // If the update value is nullish, keep the original value unless it doesn't exist
-            else if (sourceValue === undefined) {
+            else if (sourceValue === undefined || sourceValue === null) {
                 result[key] = updateValue;
             }
         });
@@ -314,10 +314,10 @@ export class DataLoader {
                             // Process each item in the collection
                             collectionData.data.forEach((item) => {
                                 const oldId = item.context[idMapping.sourceField];
-                                const newId = this.mergedUserMap.get(oldId);
+                                const newId = this.mergedUserMap.get(`${oldId}`);
                                 if (newId) {
                                     // Update context to use new user ID
-                                    item.context[idMapping.fieldToSet || targetFieldKey] = newId;
+                                    item.finalData[idMapping.fieldToSet || idMapping.sourceField] = newId;
                                 }
                             });
                         }
@@ -398,13 +398,14 @@ export class DataLoader {
                                     if (!currentDataFiltered) {
                                         // Set new data if current data is undefined
                                         collectionData.data[i].finalData[fieldToSetKey] =
-                                            Array.isArray(newData) ? newData : [newData];
+                                            Array.isArray(newData) ? newData[0] : newData;
                                     }
                                     else if (Array.isArray(newData) && newData.length > 0) {
                                         // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
+                                        // and take the first value, because it's an array and the attribute is not an array
                                         collectionData.data[i].finalData[fieldToSetKey] = [
                                             ...new Set([currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}`)),
-                                        ];
+                                        ].slice(0, 1)[0];
                                     }
                                     else if (!Array.isArray(newData) && newData !== undefined) {
                                         // Simply update the field if new data is not an array and defined
@@ -467,7 +468,6 @@ export class DataLoader {
      * @returns The transformed item with user-specific keys removed.
      */
     async prepareUserData(item, attributeMappings, primaryKeyField, newId) {
-        // Transform the item data based on the attribute mappings
         let transformedItem = this.transformData(item, attributeMappings);
         const userData = AuthUserCreateSchema.safeParse(transformedItem);
         if (!userData.success) {
@@ -477,35 +477,58 @@ export class DataLoader {
         const email = userData.data.email;
         const phone = userData.data.phone;
         let existingId;
-        // Check for duplicate email and add to emailToUserIdMap if not found
-        if (email && email.length > 0) {
-            if (this.emailToUserIdMap.has(email)) {
-                existingId = this.emailToUserIdMap.get(email);
-            }
-            else {
-                this.emailToUserIdMap.set(email, newId);
-            }
+        // Check for duplicate email and phone
+        if (email && this.emailToUserIdMap.has(email)) {
+            existingId = this.emailToUserIdMap.get(email);
         }
-        // Check for duplicate phone and add to phoneToUserIdMap if not found
-        if (phone && phone.length > 0) {
-            if (this.phoneToUserIdMap.has(phone)) {
-                existingId = this.phoneToUserIdMap.get(phone);
-            }
-            else {
-                this.phoneToUserIdMap.set(phone, newId);
-            }
+        else if (phone && this.phoneToUserIdMap.has(phone)) {
+            existingId = this.phoneToUserIdMap.get(phone);
         }
-        if (!existingId) {
-            existingId = newId;
+        else {
+            if (email)
+                this.emailToUserIdMap.set(email, newId);
+            if (phone)
+                this.phoneToUserIdMap.set(phone, newId);
         }
-        // If existingId is found, add to mergedUserMap
         if (existingId) {
             userData.data.userId = existingId;
             const mergedUsers = this.mergedUserMap.get(existingId) || [];
             mergedUsers.push(`${item[primaryKeyField]}`);
             this.mergedUserMap.set(existingId, mergedUsers);
+            const userFound = this.importMap
+                .get(this.getCollectionKey("users"))
+                ?.data.find((userDataExisting) => {
+                let userIdToMatch;
+                if (userDataExisting?.finalData?.userId) {
+                    userIdToMatch = userDataExisting?.finalData?.userId;
+                }
+                else if (userDataExisting?.finalData?.docId) {
+                    userIdToMatch = userDataExisting?.finalData?.docId;
+                }
+                else if (userDataExisting?.context?.userId) {
+                    userIdToMatch = userDataExisting.context.userId;
+                }
+                else if (userDataExisting?.context?.docId) {
+                    userIdToMatch = userDataExisting.context.docId;
+                }
+                return userIdToMatch === existingId;
+            });
+            if (userFound) {
+                userFound.finalData.userId = existingId;
+            }
+            return [
+                transformedItem,
+                existingId,
+                {
+                    rawData: userFound?.rawData,
+                    finalData: userFound?.finalData,
+                },
+            ];
+        }
+        else {
+            existingId = newId;
+            userData.data.userId = existingId;
         }
-        // Remove user-specific keys from the transformed item
         const userKeys = ["email", "phone", "name", "labels", "prefs"];
         userKeys.forEach((key) => {
             if (transformedItem.hasOwnProperty(key)) {
@@ -517,7 +540,6 @@ export class DataLoader {
             rawData: item,
             finalData: userData.data,
         };
-        // Directly update the importMap with the new user data, without pushing to usersMap.data first
         this.importMap.set(this.getCollectionKey("users"), {
             data: [...(usersMap?.data || []), userDataToAdd],
         });

package/dist/migrations/importController.js

@@ -86,44 +86,58 @@ export class ImportController {
                 const usersData = usersDataMap?.data;
                 const usersController = new UsersController(this.config, this.database);
                 if (usersData) {
-                    console.log("Found users data");
-                    const userBatchesAll = createBatches(usersData);
-                    console.log(`${userBatchesAll.length} user batches`);
-                    for (let i = 0; i < userBatchesAll.length; i++) {
-                        const userBatches = userBatchesAll[i];
-                        console.log(`Processing user batch ${i + 1} of ${userBatchesAll.length}`);
-                        const userBatchPromises = userBatches
-                            .map((userBatch) => {
-                            if (userBatch.finalData && userBatch.finalData.length > 0) {
-                                const userId = userBatch.finalData.userId;
-                                if (dataLoader.userExistsMap.has(userId)) {
-                                    // We only are storing the existing user ID's as true, so we need to check for that
-                                    if (!(dataLoader.userExistsMap.get(userId) === true)) {
-                                        const userId = userBatch.finalData.userId ||
-                                            userBatch.context.userId ||
-                                            userBatch.context.docId;
-                                        if (!userBatch.finalData.userId) {
-                                            userBatch.finalData.userId = userId;
-                                        }
-                                        return usersController
-                                            .createUserAndReturn(userBatch.finalData)
-                                            .then(() => console.log("Created user"))
-                                            .catch((error) => {
-                                            logger.error("Error creating user:", error, "\nUser data is ", userBatch.finalData);
-                                        });
-                                    }
-                                    else {
-                                        console.log("Skipped existing user: ", userId);
-                                        return Promise.resolve();
-                                    }
-                                }
+                    console.log("Found users data", usersData.length);
+                    const userDataBatches = createBatches(usersData);
+                    for (const batch of userDataBatches) {
+                        console.log("Importing users batch", batch.length);
+                        const userBatchPromises = batch
+                            .filter((item) => {
+                            let itemId;
+                            if (item.finalData.userId) {
+                                itemId = item.finalData.userId;
                             }
+                            else if (item.finalData.docId) {
+                                itemId = item.finalData.docId;
+                            }
+                            if (!itemId) {
+                                return false;
+                            }
+                            return (item &&
+                                item.finalData &&
+                                !dataLoader.userExistsMap.has(itemId));
                         })
-                            .flat();
-                        // Wait for all promises in the current user batch to resolve
-                        await Promise.allSettled(userBatchPromises);
-                        console.log(`Completed user batch ${i + 1} of ${userBatchesAll.length}`);
+                            .map((item) => {
+                            return usersController.createUserAndReturn(item.finalData);
+                        });
+                        await Promise.all(userBatchPromises);
+                        for (const item of batch) {
+                            if (item && item.finalData) {
+                                dataLoader.userExistsMap.set(item.finalData.userId ||
+                                    item.finalData.docId ||
+                                    item.context.userId ||
+                                    item.context.docId, true);
+                            }
+                        }
+                        console.log("Finished importing users batch", batch.length);
                     }
+                    // for (let i = 0; i < usersData.length; i++) {
+                    //   const user = usersData[i];
+                    //   if (user.finalData) {
+                    //     const userId =
+                    //       user.finalData.userId ||
+                    //       user.context.userId ||
+                    //       user.context.docId;
+                    //     if (!dataLoader.userExistsMap.has(userId)) {
+                    //       if (!user.finalData.userId) {
+                    //         user.finalData.userId = userId;
+                    //       }
+                    //       await usersController.createUserAndReturn(user.finalData);
+                    //       dataLoader.userExistsMap.set(userId, true);
+                    //     } else {
+                    //       console.log("Skipped existing user: ", userId);
+                    //     }
+                    //   }
+                    // }
                     console.log("Finished importing users");
                 }
             }
@@ -146,10 +160,10 @@ export class ImportController {
                 const batches = dataSplit[i];
                 console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
                 const batchPromises = batches.map((item) => {
-                    const id = item.context.docId ||
-                        item.context.userId ||
-                        item.finalData.docId ||
-                        item.finalData.userId;
+                    const id = item.finalData.docId ||
+                        item.finalData.userId ||
+                        item.context.docId ||
+                        item.context.userId;
                     if (item.finalData.hasOwnProperty("userId")) {
                         delete item.finalData.userId;
                     }
@@ -159,17 +173,7 @@ export class ImportController {
                     if (!item.finalData) {
                         return Promise.resolve();
                     }
-                    return this.database
-                        .createDocument(db.$id, collection.$id, id, item.finalData)
-                        .then(() => {
-                        processedItems++;
-                        console.log("Created item");
-                    })
-                        .catch((error) => {
-                        console.error(`Error creating item in ${collection.name}:`, error, "\nItem data is ", item.finalData);
-                        throw error;
-                        // Optionally, log the failed item for retry or review
-                    });
+                    return this.database.createDocument(db.$id, collection.$id, id, item.finalData);
                 });
                 // Wait for all promises in the current batch to resolve
                 await Promise.allSettled(batchPromises);

package/dist/migrations/users.js

@@ -115,6 +115,7 @@ export class UsersController {
             if (e instanceof Error) {
                 logger.error("FAILED CREATING USER: ", e.message);
             }
+            console.log("FAILED CREATING USER: ", e);
             throw e;
         }
     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.33",
+  "version": "0.0.34",
   "main": "src/main.ts",
   "type": "module",
   "repository": {
@@ -22,7 +22,7 @@
   ],
   "bin": {
     "appwrite-init": "./dist/init.js",
-    "appwrite-migrate": "node --max-old-space-size=16384 ./dist/main.js"
+    "appwrite-migrate": "./dist/main.js"
   },
   "scripts": {
     "build": "bun run tsc",

package/src/migrations/dataLoader.ts

@@ -124,7 +124,7 @@ export class DataLoader {
         result[key] = updateValue;
       }
       // If the update value is nullish, keep the original value unless it doesn't exist
-      else if (sourceValue === undefined) {
+      else if (sourceValue === undefined || sourceValue === null) {
         result[key] = updateValue;
       }
     });
@@ -395,11 +395,13 @@ export class DataLoader {
               // Process each item in the collection
               collectionData.data.forEach((item) => {
                 const oldId = item.context[idMapping.sourceField];
-                const newId = this.mergedUserMap.get(oldId);
+                const newId = this.mergedUserMap.get(`${oldId}`);
 
                 if (newId) {
                   // Update context to use new user ID
-                  item.context[idMapping.fieldToSet || targetFieldKey] = newId;
+                  item.finalData[
+                    idMapping.fieldToSet || idMapping.sourceField
+                  ] = newId;
                 }
               });
             }
@@ -524,16 +526,17 @@ export class DataLoader {
                   if (!currentDataFiltered) {
                     // Set new data if current data is undefined
                     collectionData.data[i].finalData[fieldToSetKey] =
-                      Array.isArray(newData) ? newData : [newData];
+                      Array.isArray(newData) ? newData[0] : newData;
                   } else if (Array.isArray(newData) && newData.length > 0) {
                     // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
+                    // and take the first value, because it's an array and the attribute is not an array
                     collectionData.data[i].finalData[fieldToSetKey] = [
                       ...new Set(
                         [currentDataFiltered, ...newData].filter(
                           (value: any) => `${value}` !== `${valueToMatch}`
                         )
                       ),
-                    ];
+                    ].slice(0, 1)[0];
                   } else if (!Array.isArray(newData) && newData !== undefined) {
                     // Simply update the field if new data is not an array and defined
                     collectionData.data[i].finalData[fieldToSetKey] = newData;
@@ -615,7 +618,6 @@ export class DataLoader {
     primaryKeyField: string,
     newId: string
   ): Promise<any> {
-    // Transform the item data based on the attribute mappings
     let transformedItem = this.transformData(item, attributeMappings);
     const userData = AuthUserCreateSchema.safeParse(transformedItem);
     if (!userData.success) {
@@ -632,48 +634,64 @@ export class DataLoader {
     const phone = userData.data.phone;
     let existingId: string | undefined;
 
-    // Check for duplicate email and add to emailToUserIdMap if not found
-    if (email && email.length > 0) {
-      if (this.emailToUserIdMap.has(email)) {
-        existingId = this.emailToUserIdMap.get(email);
-      } else {
-        this.emailToUserIdMap.set(email, newId);
-      }
-    }
-
-    // Check for duplicate phone and add to phoneToUserIdMap if not found
-    if (phone && phone.length > 0) {
-      if (this.phoneToUserIdMap.has(phone)) {
-        existingId = this.phoneToUserIdMap.get(phone);
-      } else {
-        this.phoneToUserIdMap.set(phone, newId);
-      }
-    }
-    if (!existingId) {
-      existingId = newId;
+    // Check for duplicate email and phone
+    if (email && this.emailToUserIdMap.has(email)) {
+      existingId = this.emailToUserIdMap.get(email);
+    } else if (phone && this.phoneToUserIdMap.has(phone)) {
+      existingId = this.phoneToUserIdMap.get(phone);
+    } else {
+      if (email) this.emailToUserIdMap.set(email, newId);
+      if (phone) this.phoneToUserIdMap.set(phone, newId);
     }
 
-    // If existingId is found, add to mergedUserMap
     if (existingId) {
       userData.data.userId = existingId;
       const mergedUsers = this.mergedUserMap.get(existingId) || [];
       mergedUsers.push(`${item[primaryKeyField]}`);
       this.mergedUserMap.set(existingId, mergedUsers);
+      const userFound = this.importMap
+        .get(this.getCollectionKey("users"))
+        ?.data.find((userDataExisting) => {
+          let userIdToMatch: string | undefined;
+          if (userDataExisting?.finalData?.userId) {
+            userIdToMatch = userDataExisting?.finalData?.userId;
+          } else if (userDataExisting?.finalData?.docId) {
+            userIdToMatch = userDataExisting?.finalData?.docId;
+          } else if (userDataExisting?.context?.userId) {
+            userIdToMatch = userDataExisting.context.userId;
+          } else if (userDataExisting?.context?.docId) {
+            userIdToMatch = userDataExisting.context.docId;
+          }
+          return userIdToMatch === existingId;
+        });
+      if (userFound) {
+        userFound.finalData.userId = existingId;
+      }
+      return [
+        transformedItem,
+        existingId,
+        {
+          rawData: userFound?.rawData,
+          finalData: userFound?.finalData,
+        },
+      ];
+    } else {
+      existingId = newId;
+      userData.data.userId = existingId;
     }
 
-    // Remove user-specific keys from the transformed item
     const userKeys = ["email", "phone", "name", "labels", "prefs"];
     userKeys.forEach((key) => {
       if (transformedItem.hasOwnProperty(key)) {
         delete transformedItem[key];
       }
     });
+
     const usersMap = this.importMap.get(this.getCollectionKey("users"));
     const userDataToAdd = {
       rawData: item,
       finalData: userData.data,
     };
-    // Directly update the importMap with the new user data, without pushing to usersMap.data first
     this.importMap.set(this.getCollectionKey("users"), {
       data: [...(usersMap?.data || []), userDataToAdd],
     });

package/src/migrations/importController.ts

@@ -137,53 +137,62 @@ export class ImportController {
         const usersData = usersDataMap?.data;
         const usersController = new UsersController(this.config, this.database);
         if (usersData) {
-          console.log("Found users data");
-          const userBatchesAll = createBatches(usersData);
-          console.log(`${userBatchesAll.length} user batches`);
-          for (let i = 0; i < userBatchesAll.length; i++) {
-            const userBatches = userBatchesAll[i];
-            console.log(
-              `Processing user batch ${i + 1} of ${userBatchesAll.length}`
-            );
-            const userBatchPromises = userBatches
-              .map((userBatch) => {
-                if (userBatch.finalData && userBatch.finalData.length > 0) {
-                  const userId = userBatch.finalData.userId;
-                  if (dataLoader.userExistsMap.has(userId)) {
-                    // We only are storing the existing user ID's as true, so we need to check for that
-                    if (!(dataLoader.userExistsMap.get(userId) === true)) {
-                      const userId =
-                        userBatch.finalData.userId ||
-                        userBatch.context.userId ||
-                        userBatch.context.docId;
-                      if (!userBatch.finalData.userId) {
-                        userBatch.finalData.userId = userId;
-                      }
-                      return usersController
-                        .createUserAndReturn(userBatch.finalData)
-                        .then(() => console.log("Created user"))
-                        .catch((error) => {
-                          logger.error(
-                            "Error creating user:",
-                            error,
-                            "\nUser data is ",
-                            userBatch.finalData
-                          );
-                        });
-                    } else {
-                      console.log("Skipped existing user: ", userId);
-                      return Promise.resolve();
-                    }
-                  }
+          console.log("Found users data", usersData.length);
+          const userDataBatches = createBatches(usersData);
+          for (const batch of userDataBatches) {
+            console.log("Importing users batch", batch.length);
+            const userBatchPromises = batch
+              .filter((item) => {
+                let itemId: string | undefined;
+                if (item.finalData.userId) {
+                  itemId = item.finalData.userId;
+                } else if (item.finalData.docId) {
+                  itemId = item.finalData.docId;
+                }
+                if (!itemId) {
+                  return false;
                 }
+                return (
+                  item &&
+                  item.finalData &&
+                  !dataLoader.userExistsMap.has(itemId)
+                );
               })
-              .flat();
-            // Wait for all promises in the current user batch to resolve
-            await Promise.allSettled(userBatchPromises);
-            console.log(
-              `Completed user batch ${i + 1} of ${userBatchesAll.length}`
-            );
+              .map((item) => {
+                return usersController.createUserAndReturn(item.finalData);
+              });
+            await Promise.all(userBatchPromises);
+            for (const item of batch) {
+              if (item && item.finalData) {
+                dataLoader.userExistsMap.set(
+                  item.finalData.userId ||
+                    item.finalData.docId ||
+                    item.context.userId ||
+                    item.context.docId,
+                  true
+                );
+              }
+            }
+            console.log("Finished importing users batch", batch.length);
           }
+          // for (let i = 0; i < usersData.length; i++) {
+          //   const user = usersData[i];
+          //   if (user.finalData) {
+          //     const userId =
+          //       user.finalData.userId ||
+          //       user.context.userId ||
+          //       user.context.docId;
+          //     if (!dataLoader.userExistsMap.has(userId)) {
+          //       if (!user.finalData.userId) {
+          //         user.finalData.userId = userId;
+          //       }
+          //       await usersController.createUserAndReturn(user.finalData);
+          //       dataLoader.userExistsMap.set(userId, true);
+          //     } else {
+          //       console.log("Skipped existing user: ", userId);
+          //     }
+          //   }
+          // }
           console.log("Finished importing users");
         }
       }
@@ -216,10 +225,10 @@ export class ImportController {
         console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
         const batchPromises = batches.map((item) => {
           const id =
-            item.context.docId ||
-            item.context.userId ||
             item.finalData.docId ||
-            item.finalData.userId;
+            item.finalData.userId ||
+            item.context.docId ||
+            item.context.userId;
           if (item.finalData.hasOwnProperty("userId")) {
             delete item.finalData.userId;
           }
@@ -229,22 +238,12 @@ export class ImportController {
           if (!item.finalData) {
             return Promise.resolve();
           }
-          return this.database
-            .createDocument(db.$id, collection.$id, id, item.finalData)
-            .then(() => {
-              processedItems++;
-              console.log("Created item");
-            })
-            .catch((error) => {
-              console.error(
-                `Error creating item in ${collection.name}:`,
-                error,
-                "\nItem data is ",
-                item.finalData
-              );
-              throw error;
-              // Optionally, log the failed item for retry or review
-            });
+          return this.database.createDocument(
+            db.$id,
+            collection.$id,
+            id,
+            item.finalData
+          );
         });
         // Wait for all promises in the current batch to resolve
         await Promise.allSettled(batchPromises);

package/src/migrations/users.ts

@@ -132,6 +132,7 @@ export class UsersController {
       if (e instanceof Error) {
         logger.error("FAILED CREATING USER: ", e.message);
       }
+      console.log("FAILED CREATING USER: ", e);
       throw e;
     }
   }