appwrite-utils-cli 0.0.32 → 0.0.34

package/src/migrations/dataLoader.ts

@@ -124,7 +124,7 @@ export class DataLoader {
         result[key] = updateValue;
       }
       // If the update value is nullish, keep the original value unless it doesn't exist
-      else if (sourceValue === undefined) {
+      else if (sourceValue === undefined || sourceValue === null) {
         result[key] = updateValue;
       }
     });
@@ -298,6 +298,10 @@ export class DataLoader {
         let isUsersCollection =
           this.getCollectionKey(this.config.usersCollectionName) ===
           this.getCollectionKey(collection.name);
+        const collectionDefs = collection.importDefs;
+        if (!collectionDefs || !collectionDefs.length) {
+          continue;
+        }
         // Process create and update definitions for the collection
         const createDefs = collection.importDefs.filter(
           (def: ImportDef) => def.type === "create" || !def.type
@@ -331,8 +335,8 @@ export class DataLoader {
         }
       }
       console.log("Running update references");
-      await this.dealWithMergedUsers();
-      await this.updateOldReferencesForNew();
+      this.dealWithMergedUsers();
+      this.updateOldReferencesForNew();
       console.log("Done running update references");
     }
     // for (const collection of this.config.collections) {
@@ -346,7 +350,7 @@ export class DataLoader {
     }
   }
 
-  async dealWithMergedUsers() {
+  dealWithMergedUsers() {
     const usersCollectionKey = this.getCollectionKey(
       this.config.usersCollectionName
     );
@@ -357,7 +361,11 @@ export class DataLoader {
     // Collect primary key fields from the users collection definitions
     this.config.collections.forEach((collection) => {
       if (this.getCollectionKey(collection.name) === usersCollectionKey) {
-        collection.importDefs.forEach((importDef) => {
+        const collectionImportDefs = collection.importDefs;
+        if (!collectionImportDefs || !collectionImportDefs.length) {
+          return;
+        }
+        collectionImportDefs.forEach((importDef) => {
           if (importDef.primaryKeyField) {
             usersCollectionPrimaryKeyFields.add(importDef.primaryKeyField);
           }
@@ -371,23 +379,29 @@ export class DataLoader {
         this.getCollectionKey(collection.name)
       );
       if (!collectionData || !collectionData.data) return;
-
-      collection.importDefs.forEach((importDef) => {
+      const collectionImportDefs = collection.importDefs;
+      if (!collectionImportDefs || !collectionImportDefs.length) {
+        return;
+      }
+      collectionImportDefs.forEach((importDef) => {
         importDef.idMappings?.forEach((idMapping) => {
           if (
             this.getCollectionKey(idMapping.targetCollection) ===
             usersCollectionKey
           ) {
-            if (usersCollectionPrimaryKeyFields.has(idMapping.targetField)) {
+            const targetFieldKey =
+              idMapping.targetFieldToMatch || idMapping.targetField;
+            if (usersCollectionPrimaryKeyFields.has(targetFieldKey)) {
               // Process each item in the collection
               collectionData.data.forEach((item) => {
                 const oldId = item.context[idMapping.sourceField];
-                const newId = this.mergedUserMap.get(oldId);
+                const newId = this.mergedUserMap.get(`${oldId}`);
 
                 if (newId) {
                   // Update context to use new user ID
-                  item.context[idMapping.fieldToSet || idMapping.targetField] =
-                    newId;
+                  item.finalData[
+                    idMapping.fieldToSet || idMapping.sourceField
+                  ] = newId;
                 }
               });
             }
@@ -401,6 +415,7 @@ export class DataLoader {
     if (!this.config.collections) {
       return;
     }
+
     for (const collectionConfig of this.config.collections) {
       const collectionKey = this.getCollectionKey(collectionConfig.name);
       const collectionData = this.importMap.get(collectionKey);
@@ -424,29 +439,41 @@ export class DataLoader {
                 );
                 const fieldToSetKey =
                   idMapping.fieldToSet || idMapping.sourceField;
+                const targetFieldKey =
+                  idMapping.targetFieldToMatch || idMapping.targetField;
                 const valueToMatch =
                   collectionData.data[i].context[idMapping.sourceField];
 
+                // Skip if value to match is missing or empty
                 if (!valueToMatch || _.isEmpty(valueToMatch)) continue;
 
+                const isFieldToSetArray = collectionConfig.attributes.find(
+                  (attribute) => attribute.key === fieldToSetKey
+                )?.array;
+
                 const targetCollectionData =
                   this.importMap.get(targetCollectionKey);
                 if (!targetCollectionData || !targetCollectionData.data)
                   continue;
 
-                const foundData = targetCollectionData.data.filter((data) => {
-                  const targetValue = data.context[idMapping.targetField];
-                  const isMatch = `${targetValue}` === `${valueToMatch}`;
-                  // Debugging output to understand what's being compared
-                  logger.warn(
-                    `Comparing target: ${targetValue} with match: ${valueToMatch} - Result: ${isMatch}`
-                  );
-                  return isMatch;
-                });
+                // Find matching data in the target collection
+                const foundData = targetCollectionData.data.filter(
+                  ({ context }) => {
+                    const targetValue = context[targetFieldKey];
+                    const isMatch = `${targetValue}` === `${valueToMatch}`;
+                    // Ensure the targetValue is defined and not null
+                    return (
+                      isMatch &&
+                      targetValue !== undefined &&
+                      targetValue !== null
+                    );
+                  }
+                );
 
+                // Log and skip if no matching data found
                 if (!foundData.length) {
                   console.log(
-                    `No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey}`
+                    `No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`
                   );
                   logger.error(
                     `No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(
@@ -460,143 +487,72 @@ export class DataLoader {
 
                 needsUpdate = true;
 
-                // Properly handle arrays and non-arrays
-                if (
-                  Array.isArray(collectionData.data[i].finalData[fieldToSetKey])
-                ) {
-                  collectionData.data[i].finalData[fieldToSetKey] =
-                    foundData.map((data) => data.finalData);
-                } else {
-                  collectionData.data[i].finalData[fieldToSetKey] =
-                    foundData[0].finalData;
-                }
-              }
-            }
-          }
-        }
-      }
-
-      if (needsUpdate) {
-        this.importMap.set(collectionKey, collectionData);
-      }
-    }
-  }
-
-  async updateReferencesInRelatedCollections() {
-    if (!this.config.collections) {
-      return;
-    }
-    // Iterate over each collection configuration
-    for (const collectionConfig of this.config.collections) {
-      const collectionKey = this.getCollectionKey(collectionConfig.name);
-      const collectionData = this.importMap.get(collectionKey);
-
-      if (!collectionData || !collectionData.data) continue;
-
-      console.log(
-        `Updating references for collection: ${collectionConfig.name}`
-      );
+                const getCurrentDataFiltered = (currentData: any) => {
+                  if (Array.isArray(currentData.finalData[fieldToSetKey])) {
+                    return currentData.finalData[fieldToSetKey].filter(
+                      (data: any) => `${data}` !== `${valueToMatch}`
+                    );
+                  }
+                  return currentData.finalData[fieldToSetKey];
+                };
 
-      // Iterate over each data item in the current collection
-      for (const item of collectionData.data) {
-        let needsUpdate = false;
+                // Get the current data to be updated
+                const currentDataFiltered = getCurrentDataFiltered(
+                  collectionData.data[i]
+                );
 
-        // Check if the current collection has import definitions with idMappings
-        if (collectionConfig.importDefs) {
-          for (const importDef of collectionConfig.importDefs) {
-            if (importDef.idMappings) {
-              // Iterate over each idMapping defined for the current import definition
-              for (const idMapping of importDef.idMappings) {
-                const oldIds = Array.isArray(
-                  item.context[idMapping.sourceField]
-                )
-                  ? item.context[idMapping.sourceField]
-                  : [item.context[idMapping.sourceField]];
-                const resolvedNewIds: string[] = [];
-
-                oldIds.forEach((oldId: any) => {
-                  // Attempt to find a new ID for the old ID
-                  let newIdForOldId = this.findNewIdForOldId(
-                    oldId,
-                    idMapping,
-                    importDef
-                  );
+                // Extract the new data to set
+                const newData = foundData.map(
+                  (data) => data.context[idMapping.targetField]
+                );
 
-                  if (
-                    newIdForOldId &&
-                    !resolvedNewIds.includes(newIdForOldId)
-                  ) {
-                    resolvedNewIds.push(newIdForOldId);
+                // Handle cases where current data is an array
+                if (isFieldToSetArray) {
+                  if (!currentDataFiltered) {
+                    // Set new data if current data is undefined
+                    collectionData.data[i].finalData[fieldToSetKey] =
+                      Array.isArray(newData) ? newData : [newData];
                   } else {
-                    logger.error(
-                      `No new ID found for old ID ${oldId} in collection ${collectionConfig.name}`
-                    );
+                    // Merge arrays if new data is non-empty array and filter for uniqueness
+                    collectionData.data[i].finalData[fieldToSetKey] = [
+                      ...new Set(
+                        [...currentDataFiltered, ...newData].filter(
+                          (value: any) => `${value}` !== `${valueToMatch}`
+                        )
+                      ),
+                    ];
+                  }
+                } else {
+                  if (!currentDataFiltered) {
+                    // Set new data if current data is undefined
+                    collectionData.data[i].finalData[fieldToSetKey] =
+                      Array.isArray(newData) ? newData[0] : newData;
+                  } else if (Array.isArray(newData) && newData.length > 0) {
+                    // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
+                    // and take the first value, because it's an array and the attribute is not an array
+                    collectionData.data[i].finalData[fieldToSetKey] = [
+                      ...new Set(
+                        [currentDataFiltered, ...newData].filter(
+                          (value: any) => `${value}` !== `${valueToMatch}`
+                        )
+                      ),
+                    ].slice(0, 1)[0];
+                  } else if (!Array.isArray(newData) && newData !== undefined) {
+                    // Simply update the field if new data is not an array and defined
+                    collectionData.data[i].finalData[fieldToSetKey] = newData;
                   }
-                });
-
-                if (resolvedNewIds.length) {
-                  const targetField =
-                    idMapping.fieldToSet || idMapping.targetField;
-                  const isArray = collectionConfig.attributes.some(
-                    (attribute) =>
-                      attribute.key === targetField && attribute.array
-                  );
-
-                  // Set the target field based on whether it's an array or single value
-                  item.finalData[targetField] = isArray
-                    ? resolvedNewIds
-                    : resolvedNewIds[0];
-                  needsUpdate = true;
                 }
               }
             }
           }
         }
-
-        // Update the importMap if changes were made to the item
-        if (needsUpdate) {
-          this.importMap.set(collectionKey, collectionData);
-          logger.info(
-            `Updated item: ${JSON.stringify(item.finalData, undefined, 2)}`
-          );
-        }
-      }
-    }
-  }
-
-  findNewIdForOldId(oldId: string, idMapping: IdMapping, importDef: ImportDef) {
-    // First, check if this ID mapping is related to the users collection.
-    const targetCollectionKey = this.getCollectionKey(
-      idMapping.targetCollection
-    );
-    const isUsersCollection =
-      targetCollectionKey ===
-      this.getCollectionKey(this.config.usersCollectionName);
-
-    // If handling users, check the mergedUserMap for any existing new ID.
-    if (isUsersCollection) {
-      for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
-        if (oldIds.includes(oldId)) {
-          return newUserId;
-        }
       }
-    }
 
-    // If not a user or no merged ID found, check the regular ID mapping from old to new.
-    const targetCollectionData = this.importMap.get(targetCollectionKey);
-    if (targetCollectionData) {
-      const foundEntry = targetCollectionData.data.find(
-        (entry) => entry.context[importDef.primaryKeyField] === oldId
-      );
-      if (foundEntry) {
-        return foundEntry.context.docId; // Assuming `docId` stores the new ID after import
+      // Update the import map if any changes were made
+      if (needsUpdate) {
+        this.importMap.set(collectionKey, collectionData);
       }
     }
-
-    logger.error(
-      `No corresponding new ID found for ${oldId} in ${targetCollectionKey}`
-    );
-    return null; // Return null if no new ID is found
   }
 
   private writeMapsToJsonFile() {
@@ -662,7 +618,6 @@ export class DataLoader {
     primaryKeyField: string,
     newId: string
   ): Promise<any> {
-    // Transform the item data based on the attribute mappings
     let transformedItem = this.transformData(item, attributeMappings);
     const userData = AuthUserCreateSchema.safeParse(transformedItem);
     if (!userData.success) {
@@ -679,48 +634,64 @@ export class DataLoader {
     const phone = userData.data.phone;
     let existingId: string | undefined;
 
-    // Check for duplicate email and add to emailToUserIdMap if not found
-    if (email && email.length > 0) {
-      if (this.emailToUserIdMap.has(email)) {
-        existingId = this.emailToUserIdMap.get(email);
-      } else {
-        this.emailToUserIdMap.set(email, newId);
-      }
+    // Check for duplicate email and phone
+    if (email && this.emailToUserIdMap.has(email)) {
+      existingId = this.emailToUserIdMap.get(email);
+    } else if (phone && this.phoneToUserIdMap.has(phone)) {
+      existingId = this.phoneToUserIdMap.get(phone);
+    } else {
+      if (email) this.emailToUserIdMap.set(email, newId);
+      if (phone) this.phoneToUserIdMap.set(phone, newId);
     }
 
-    // Check for duplicate phone and add to phoneToUserIdMap if not found
-    if (phone && phone.length > 0) {
-      if (this.phoneToUserIdMap.has(phone)) {
-        existingId = this.phoneToUserIdMap.get(phone);
-      } else {
-        this.phoneToUserIdMap.set(phone, newId);
-      }
-    }
-    if (!existingId) {
-      existingId = newId;
-    }
-
-    // If existingId is found, add to mergedUserMap
     if (existingId) {
       userData.data.userId = existingId;
       const mergedUsers = this.mergedUserMap.get(existingId) || [];
       mergedUsers.push(`${item[primaryKeyField]}`);
       this.mergedUserMap.set(existingId, mergedUsers);
+      const userFound = this.importMap
+        .get(this.getCollectionKey("users"))
+        ?.data.find((userDataExisting) => {
+          let userIdToMatch: string | undefined;
+          if (userDataExisting?.finalData?.userId) {
+            userIdToMatch = userDataExisting?.finalData?.userId;
+          } else if (userDataExisting?.finalData?.docId) {
+            userIdToMatch = userDataExisting?.finalData?.docId;
+          } else if (userDataExisting?.context?.userId) {
+            userIdToMatch = userDataExisting.context.userId;
+          } else if (userDataExisting?.context?.docId) {
+            userIdToMatch = userDataExisting.context.docId;
+          }
+          return userIdToMatch === existingId;
+        });
+      if (userFound) {
+        userFound.finalData.userId = existingId;
+      }
+      return [
+        transformedItem,
+        existingId,
+        {
+          rawData: userFound?.rawData,
+          finalData: userFound?.finalData,
+        },
+      ];
+    } else {
+      existingId = newId;
+      userData.data.userId = existingId;
     }
 
-    // Remove user-specific keys from the transformed item
     const userKeys = ["email", "phone", "name", "labels", "prefs"];
     userKeys.forEach((key) => {
       if (transformedItem.hasOwnProperty(key)) {
         delete transformedItem[key];
       }
     });
+
     const usersMap = this.importMap.get(this.getCollectionKey("users"));
     const userDataToAdd = {
       rawData: item,
       finalData: userData.data,
     };
-    // Directly update the importMap with the new user data, without pushing to usersMap.data first
     this.importMap.set(this.getCollectionKey("users"), {
       data: [...(usersMap?.data || []), userDataToAdd],
     });
@@ -808,14 +779,14 @@ export class DataLoader {
       if (!existingId) {
         // No existing user ID, generate a new unique ID
         existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
-        transformedItem.userId = existingId; // Assign the new ID to the transformed data's userId field
+        transformedItem.docId = existingId; // Assign the new ID to the transformed data's docId field
       }
 
       // Create a context object for the item, including the new ID
       let context = this.createContext(db, collection, item, existingId);
 
       // Merge the transformed data into the context
-      context = { ...context, ...transformedItem };
+      context = { ...context, ...transformedItem, ...userData.finalData };
 
       // If a primary key field is defined, handle the ID mapping and check for duplicates
       if (importDef.primaryKeyField) {

package/src/migrations/importController.ts

@@ -137,53 +137,62 @@ export class ImportController {
         const usersData = usersDataMap?.data;
         const usersController = new UsersController(this.config, this.database);
         if (usersData) {
-          console.log("Found users data");
-          const userBatchesAll = createBatches(usersData);
-          console.log(`${userBatchesAll.length} user batches`);
-          for (let i = 0; i < userBatchesAll.length; i++) {
-            const userBatches = userBatchesAll[i];
-            console.log(
-              `Processing user batch ${i + 1} of ${userBatchesAll.length}`
-            );
-            const userBatchPromises = userBatches
-              .map((userBatch) => {
-                if (userBatch.finalData && userBatch.finalData.length > 0) {
-                  const userId = userBatch.finalData.userId;
-                  if (dataLoader.userExistsMap.has(userId)) {
-                    // We only are storing the existing user ID's as true, so we need to check for that
-                    if (!(dataLoader.userExistsMap.get(userId) === true)) {
-                      const userId =
-                        userBatch.finalData.userId ||
-                        userBatch.context.userId ||
-                        userBatch.context.docId;
-                      if (!userBatch.finalData.userId) {
-                        userBatch.finalData.userId = userId;
-                      }
-                      return usersController
-                        .createUserAndReturn(userBatch.finalData)
-                        .then(() => console.log("Created user"))
-                        .catch((error) => {
-                          logger.error(
-                            "Error creating user:",
-                            error,
-                            "\nUser data is ",
-                            userBatch.finalData
-                          );
-                        });
-                    } else {
-                      console.log("Skipped existing user: ", userId);
-                      return Promise.resolve();
-                    }
-                  }
+          console.log("Found users data", usersData.length);
+          const userDataBatches = createBatches(usersData);
+          for (const batch of userDataBatches) {
+            console.log("Importing users batch", batch.length);
+            const userBatchPromises = batch
+              .filter((item) => {
+                let itemId: string | undefined;
+                if (item.finalData.userId) {
+                  itemId = item.finalData.userId;
+                } else if (item.finalData.docId) {
+                  itemId = item.finalData.docId;
+                }
+                if (!itemId) {
+                  return false;
                 }
+                return (
+                  item &&
+                  item.finalData &&
+                  !dataLoader.userExistsMap.has(itemId)
+                );
               })
-              .flat();
-            // Wait for all promises in the current user batch to resolve
-            await Promise.allSettled(userBatchPromises);
-            console.log(
-              `Completed user batch ${i + 1} of ${userBatchesAll.length}`
-            );
+              .map((item) => {
+                return usersController.createUserAndReturn(item.finalData);
+              });
+            await Promise.all(userBatchPromises);
+            for (const item of batch) {
+              if (item && item.finalData) {
+                dataLoader.userExistsMap.set(
+                  item.finalData.userId ||
+                    item.finalData.docId ||
+                    item.context.userId ||
+                    item.context.docId,
+                  true
+                );
+              }
+            }
+            console.log("Finished importing users batch", batch.length);
           }
+          // for (let i = 0; i < usersData.length; i++) {
+          //   const user = usersData[i];
+          //   if (user.finalData) {
+          //     const userId =
+          //       user.finalData.userId ||
+          //       user.context.userId ||
+          //       user.context.docId;
+          //     if (!dataLoader.userExistsMap.has(userId)) {
+          //       if (!user.finalData.userId) {
+          //         user.finalData.userId = userId;
+          //       }
+          //       await usersController.createUserAndReturn(user.finalData);
+          //       dataLoader.userExistsMap.set(userId, true);
+          //     } else {
+          //       console.log("Skipped existing user: ", userId);
+          //     }
+          //   }
+          // }
           console.log("Finished importing users");
         }
       }
@@ -216,10 +225,10 @@ export class ImportController {
         console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
         const batchPromises = batches.map((item) => {
           const id =
-            item.context.docId ||
-            item.context.userId ||
             item.finalData.docId ||
-            item.finalData.userId;
+            item.finalData.userId ||
+            item.context.docId ||
+            item.context.userId;
           if (item.finalData.hasOwnProperty("userId")) {
             delete item.finalData.userId;
           }
@@ -229,22 +238,12 @@ export class ImportController {
           if (!item.finalData) {
             return Promise.resolve();
           }
-          return this.database
-            .createDocument(db.$id, collection.$id, id, item.finalData)
-            .then(() => {
-              processedItems++;
-              console.log("Created item");
-            })
-            .catch((error) => {
-              console.error(
-                `Error creating item in ${collection.name}:`,
-                error,
-                "\nItem data is ",
-                item.finalData
-              );
-              throw error;
-              // Optionally, log the failed item for retry or review
-            });
+          return this.database.createDocument(
+            db.$id,
+            collection.$id,
+            id,
+            item.finalData
+          );
         });
         // Wait for all promises in the current batch to resolve
         await Promise.allSettled(batchPromises);