appwrite-utils-cli 0.0.265 → 0.0.267

package/dist/migrations/collections.js

@@ -148,7 +148,7 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
                 const foundColl = deletedCollections.find((coll) => coll.collectionName.toLowerCase().trim().replace(" ", "") ===
                     collection.name.toLowerCase().trim().replace(" ", ""));
                 if (foundColl) {
-                    const collectionId = foundColl.collectionId || ID.unique();
+                    const collectionId = collection.$id || foundColl.collectionId || ID.unique();
                     console.log(`Processing collection: ${collection.name} with ID: ${collectionId}`);
                     collectionToUse = await database.createCollection(databaseId, collectionId, collection.name, permissions, collection.documentSecurity, collection.enabled);
                     nameToIdMapping.set(collection.name, collectionToUse.$id);

package/dist/migrations/dataLoader.d.ts

@@ -1589,6 +1589,8 @@ export declare class DataLoader {
     setupMaps(dbId: string): Promise<void>;
     getAllUsers(): Promise<import("node-appwrite").Models.User<import("node-appwrite").Models.Preferences>[]>;
     start(dbId: string): Promise<void>;
+    dealWithMergedUsers(): Promise<void>;
+    updateOldReferencesForNew(): Promise<void>;
     updateReferencesInRelatedCollections(): Promise<void>;
     findNewIdForOldId(oldId: string, idMapping: IdMapping, importDef: ImportDef): any;
     private writeMapsToJsonFile;

package/dist/migrations/dataLoader.js

@@ -252,7 +252,8 @@ export class DataLoader {
                 }
             }
             console.log("Running update references");
-            await this.updateReferencesInRelatedCollections();
+            await this.dealWithMergedUsers();
+            await this.updateOldReferencesForNew();
             console.log("Done running update references");
         }
         // for (const collection of this.config.collections) {
@@ -265,6 +266,99 @@ export class DataLoader {
             this.writeMapsToJsonFile();
         }
     }
+    async dealWithMergedUsers() {
+        const usersCollectionKey = this.getCollectionKey(this.config.usersCollectionName);
+        const usersCollectionPrimaryKeyFields = new Set();
+        // Collect primary key fields from the users collection definitions
+        this.config.collections.forEach((collection) => {
+            if (this.getCollectionKey(collection.name) === usersCollectionKey) {
+                collection.importDefs.forEach((importDef) => {
+                    if (importDef.primaryKeyField) {
+                        usersCollectionPrimaryKeyFields.add(importDef.primaryKeyField);
+                    }
+                });
+            }
+        });
+        // Iterate over all collections to update references based on merged users
+        this.config.collections.forEach((collection) => {
+            const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
+            if (!collectionData || !collectionData.data)
+                return;
+            collection.importDefs.forEach((importDef) => {
+                importDef.idMappings?.forEach((idMapping) => {
+                    if (this.getCollectionKey(idMapping.targetCollection) ===
+                        usersCollectionKey) {
+                        if (usersCollectionPrimaryKeyFields.has(idMapping.targetField)) {
+                            // Process each item in the collection
+                            collectionData.data.forEach((item) => {
+                                const oldId = item.context[idMapping.sourceField];
+                                const newId = this.mergedUserMap.get(oldId);
+                                if (newId) {
+                                    // Update context to use new user ID
+                                    item.context[idMapping.fieldToSet || idMapping.targetField] =
+                                        newId;
+                                }
+                            });
+                        }
+                    }
+                });
+            });
+        });
+    }
+    async updateOldReferencesForNew() {
+        for (const collectionConfig of this.config.collections) {
+            const collectionKey = this.getCollectionKey(collectionConfig.name);
+            const collectionData = this.importMap.get(collectionKey);
+            if (!collectionData || !collectionData.data)
+                continue;
+            console.log(`Updating references for collection: ${collectionConfig.name}`);
+            let needsUpdate = false;
+            // Iterate over each data item in the current collection
+            for (let i = 0; i < collectionData.data.length; i++) {
+                if (collectionConfig.importDefs) {
+                    for (const importDef of collectionConfig.importDefs) {
+                        if (importDef.idMappings) {
+                            for (const idMapping of importDef.idMappings) {
+                                const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
+                                const fieldToSetKey = idMapping.fieldToSet || idMapping.sourceField;
+                                const valueToMatch = collectionData.data[i].context[idMapping.sourceField];
+                                if (!valueToMatch || _.isEmpty(valueToMatch))
+                                    continue;
+                                const targetCollectionData = this.importMap.get(targetCollectionKey);
+                                if (!targetCollectionData || !targetCollectionData.data)
+                                    continue;
+                                const foundData = targetCollectionData.data.filter((data) => {
+                                    const targetValue = data.context[idMapping.targetField];
+                                    const isMatch = `${targetValue}` === `${valueToMatch}`;
+                                    // Debugging output to understand what's being compared
+                                    logger.warn(`Comparing target: ${targetValue} with match: ${valueToMatch} - Result: ${isMatch}`);
+                                    return isMatch;
+                                });
+                                if (!foundData.length) {
+                                    console.log(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey}`);
+                                    logger.error(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
+                                    continue;
+                                }
+                                needsUpdate = true;
+                                // Properly handle arrays and non-arrays
+                                if (Array.isArray(collectionData.data[i].finalData[fieldToSetKey])) {
+                                    collectionData.data[i].finalData[fieldToSetKey] =
+                                        foundData.map((data) => data.finalData);
+                                }
+                                else {
+                                    collectionData.data[i].finalData[fieldToSetKey] =
+                                        foundData[0].finalData;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            if (needsUpdate) {
+                this.importMap.set(collectionKey, collectionData);
+            }
+        }
+    }
     async updateReferencesInRelatedCollections() {
         // Iterate over each collection configuration
         for (const collectionConfig of this.config.collections) {
@@ -289,7 +383,8 @@ export class DataLoader {
                                 oldIds.forEach((oldId) => {
                                     // Attempt to find a new ID for the old ID
                                     let newIdForOldId = this.findNewIdForOldId(oldId, idMapping, importDef);
-                                    if (newIdForOldId && !resolvedNewIds.includes(newIdForOldId)) {
+                                    if (newIdForOldId &&
+                                        !resolvedNewIds.includes(newIdForOldId)) {
                                         resolvedNewIds.push(newIdForOldId);
                                     }
                                     else {
@@ -298,9 +393,11 @@ export class DataLoader {
                                 });
                                 if (resolvedNewIds.length) {
                                     const targetField = idMapping.fieldToSet || idMapping.targetField;
-                                    const isArray = collectionConfig.attributes.some(attribute => attribute.key === targetField && attribute.array);
+                                    const isArray = collectionConfig.attributes.some((attribute) => attribute.key === targetField && attribute.array);
                                     // Set the target field based on whether it's an array or single value
-                                    item.finalData[targetField] = isArray ? resolvedNewIds : resolvedNewIds[0];
+                                    item.finalData[targetField] = isArray
+                                        ? resolvedNewIds
+                                        : resolvedNewIds[0];
                                     needsUpdate = true;
                                 }
                             }
@@ -318,7 +415,8 @@ export class DataLoader {
     findNewIdForOldId(oldId, idMapping, importDef) {
         // First, check if this ID mapping is related to the users collection.
         const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
-        const isUsersCollection = targetCollectionKey === this.getCollectionKey(this.config.usersCollectionName);
+        const isUsersCollection = targetCollectionKey ===
+            this.getCollectionKey(this.config.usersCollectionName);
         // If handling users, check the mergedUserMap for any existing new ID.
         if (isUsersCollection) {
             for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
@@ -330,7 +428,7 @@ export class DataLoader {
         // If not a user or no merged ID found, check the regular ID mapping from old to new.
         const targetCollectionData = this.importMap.get(targetCollectionKey);
         if (targetCollectionData) {
-            const foundEntry = targetCollectionData.data.find(entry => entry.context[importDef.primaryKeyField] === oldId);
+            const foundEntry = targetCollectionData.data.find((entry) => entry.context[importDef.primaryKeyField] === oldId);
             if (foundEntry) {
                 return foundEntry.context.docId; // Assuming `docId` stores the new ID after import
             }

package/dist/migrations/indexes.js

@@ -8,12 +8,6 @@ export const createOrUpdateIndex = async (dbId, db, collectionId, index) => {
     if (existingIndex.total > 0) {
         await db.deleteIndex(dbId, collectionId, existingIndex.indexes[0].key);
     }
-    if (index.type === "fulltext" && index.attributes.length > 1) {
-        throw new Error(`Fulltext index can only be created on a single attribute. Index: ${index.key}`);
-    }
-    else if (index.type === "fulltext") {
-        index.attributes = [index.attributes[0]];
-    }
     const newIndex = await db.createIndex(dbId, collectionId, index.key, index.type, index.attributes, index.orders);
     return newIndex;
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.265",
+  "version": "0.0.267",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/collections.ts

@@ -198,7 +198,8 @@ export const createOrUpdateCollections = async (
             collection.name.toLowerCase().trim().replace(" ", "")
         );
         if (foundColl) {
-          const collectionId = foundColl.collectionId || ID.unique();
+          const collectionId =
+            collection.$id || foundColl.collectionId || ID.unique();
           console.log(
             `Processing collection: ${collection.name} with ID: ${collectionId}`
           );

package/src/migrations/dataLoader.ts

@@ -325,7 +325,8 @@ export class DataLoader {
         }
       }
       console.log("Running update references");
-      await this.updateReferencesInRelatedCollections();
+      await this.dealWithMergedUsers();
+      await this.updateOldReferencesForNew();
       console.log("Done running update references");
     }
     // for (const collection of this.config.collections) {
@@ -339,98 +340,250 @@ export class DataLoader {
     }
   }
 
+  async dealWithMergedUsers() {
+    const usersCollectionKey = this.getCollectionKey(
+      this.config.usersCollectionName
+    );
+    const usersCollectionPrimaryKeyFields = new Set();
+
+    // Collect primary key fields from the users collection definitions
+    this.config.collections.forEach((collection) => {
+      if (this.getCollectionKey(collection.name) === usersCollectionKey) {
+        collection.importDefs.forEach((importDef) => {
+          if (importDef.primaryKeyField) {
+            usersCollectionPrimaryKeyFields.add(importDef.primaryKeyField);
+          }
+        });
+      }
+    });
+
+    // Iterate over all collections to update references based on merged users
+    this.config.collections.forEach((collection) => {
+      const collectionData = this.importMap.get(
+        this.getCollectionKey(collection.name)
+      );
+      if (!collectionData || !collectionData.data) return;
+
+      collection.importDefs.forEach((importDef) => {
+        importDef.idMappings?.forEach((idMapping) => {
+          if (
+            this.getCollectionKey(idMapping.targetCollection) ===
+            usersCollectionKey
+          ) {
+            if (usersCollectionPrimaryKeyFields.has(idMapping.targetField)) {
+              // Process each item in the collection
+              collectionData.data.forEach((item) => {
+                const oldId = item.context[idMapping.sourceField];
+                const newId = this.mergedUserMap.get(oldId);
+
+                if (newId) {
+                  // Update context to use new user ID
+                  item.context[idMapping.fieldToSet || idMapping.targetField] =
+                    newId;
+                }
+              });
+            }
+          }
+        });
+      });
+    });
+  }
+
+  async updateOldReferencesForNew() {
+    for (const collectionConfig of this.config.collections) {
+      const collectionKey = this.getCollectionKey(collectionConfig.name);
+      const collectionData = this.importMap.get(collectionKey);
+
+      if (!collectionData || !collectionData.data) continue;
+
+      console.log(
+        `Updating references for collection: ${collectionConfig.name}`
+      );
+
+      let needsUpdate = false;
+
+      // Iterate over each data item in the current collection
+      for (let i = 0; i < collectionData.data.length; i++) {
+        if (collectionConfig.importDefs) {
+          for (const importDef of collectionConfig.importDefs) {
+            if (importDef.idMappings) {
+              for (const idMapping of importDef.idMappings) {
+                const targetCollectionKey = this.getCollectionKey(
+                  idMapping.targetCollection
+                );
+                const fieldToSetKey =
+                  idMapping.fieldToSet || idMapping.sourceField;
+                const valueToMatch =
+                  collectionData.data[i].context[idMapping.sourceField];
+
+                if (!valueToMatch || _.isEmpty(valueToMatch)) continue;
+
+                const targetCollectionData =
+                  this.importMap.get(targetCollectionKey);
+                if (!targetCollectionData || !targetCollectionData.data)
+                  continue;
+
+                const foundData = targetCollectionData.data.filter((data) => {
+                  const targetValue = data.context[idMapping.targetField];
+                  const isMatch = `${targetValue}` === `${valueToMatch}`;
+                  // Debugging output to understand what's being compared
+                  logger.warn(
+                    `Comparing target: ${targetValue} with match: ${valueToMatch} - Result: ${isMatch}`
+                  );
+                  return isMatch;
+                });
+
+                if (!foundData.length) {
+                  console.log(
+                    `No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey}`
+                  );
+                  logger.error(
+                    `No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(
+                      idMapping,
+                      null,
+                      2
+                    )}`
+                  );
+                  continue;
+                }
+
+                needsUpdate = true;
+
+                // Properly handle arrays and non-arrays
+                if (
+                  Array.isArray(collectionData.data[i].finalData[fieldToSetKey])
+                ) {
+                  collectionData.data[i].finalData[fieldToSetKey] =
+                    foundData.map((data) => data.finalData);
+                } else {
+                  collectionData.data[i].finalData[fieldToSetKey] =
+                    foundData[0].finalData;
+                }
+              }
+            }
+          }
+        }
+      }
+
+      if (needsUpdate) {
+        this.importMap.set(collectionKey, collectionData);
+      }
+    }
+  }
+
   async updateReferencesInRelatedCollections() {
     // Iterate over each collection configuration
     for (const collectionConfig of this.config.collections) {
-        const collectionKey = this.getCollectionKey(collectionConfig.name);
-        const collectionData = this.importMap.get(collectionKey);
-
-        if (!collectionData || !collectionData.data) continue;
-
-        console.log(`Updating references for collection: ${collectionConfig.name}`);
-
-        // Iterate over each data item in the current collection
-        for (const item of collectionData.data) {
-            let needsUpdate = false;
-
-            // Check if the current collection has import definitions with idMappings
-            if (collectionConfig.importDefs) {
-                for (const importDef of collectionConfig.importDefs) {
-                    if (importDef.idMappings) {
-                        // Iterate over each idMapping defined for the current import definition
-                        for (const idMapping of importDef.idMappings) {
-                            const oldIds = Array.isArray(item.context[idMapping.sourceField])
-                                ? item.context[idMapping.sourceField]
-                                : [item.context[idMapping.sourceField]];
-                            const resolvedNewIds: string[] = [];
-
-                            oldIds.forEach((oldId: any) => {
-                                // Attempt to find a new ID for the old ID
-                                let newIdForOldId = this.findNewIdForOldId(
-                                    oldId,
-                                    idMapping,
-                                    importDef
-                                );
-
-                                if (newIdForOldId && !resolvedNewIds.includes(newIdForOldId)) {
-                                    resolvedNewIds.push(newIdForOldId);
-                                } else {
-                                    logger.error(`No new ID found for old ID ${oldId} in collection ${collectionConfig.name}`);
-                                }
-                            });
-
-                            if (resolvedNewIds.length) {
-                                const targetField = idMapping.fieldToSet || idMapping.targetField;
-                                const isArray = collectionConfig.attributes.some(
-                                    attribute => attribute.key === targetField && attribute.array
-                                );
-
-                                // Set the target field based on whether it's an array or single value
-                                item.finalData[targetField] = isArray ? resolvedNewIds : resolvedNewIds[0];
-                                needsUpdate = true;
-                            }
-                        }
-                    }
+      const collectionKey = this.getCollectionKey(collectionConfig.name);
+      const collectionData = this.importMap.get(collectionKey);
+
+      if (!collectionData || !collectionData.data) continue;
+
+      console.log(
+        `Updating references for collection: ${collectionConfig.name}`
+      );
+
+      // Iterate over each data item in the current collection
+      for (const item of collectionData.data) {
+        let needsUpdate = false;
+
+        // Check if the current collection has import definitions with idMappings
+        if (collectionConfig.importDefs) {
+          for (const importDef of collectionConfig.importDefs) {
+            if (importDef.idMappings) {
+              // Iterate over each idMapping defined for the current import definition
+              for (const idMapping of importDef.idMappings) {
+                const oldIds = Array.isArray(
+                  item.context[idMapping.sourceField]
+                )
+                  ? item.context[idMapping.sourceField]
+                  : [item.context[idMapping.sourceField]];
+                const resolvedNewIds: string[] = [];
+
+                oldIds.forEach((oldId: any) => {
+                  // Attempt to find a new ID for the old ID
+                  let newIdForOldId = this.findNewIdForOldId(
+                    oldId,
+                    idMapping,
+                    importDef
+                  );
+
+                  if (
+                    newIdForOldId &&
+                    !resolvedNewIds.includes(newIdForOldId)
+                  ) {
+                    resolvedNewIds.push(newIdForOldId);
+                  } else {
+                    logger.error(
+                      `No new ID found for old ID ${oldId} in collection ${collectionConfig.name}`
+                    );
+                  }
+                });
+
+                if (resolvedNewIds.length) {
+                  const targetField =
+                    idMapping.fieldToSet || idMapping.targetField;
+                  const isArray = collectionConfig.attributes.some(
+                    (attribute) =>
+                      attribute.key === targetField && attribute.array
+                  );
+
+                  // Set the target field based on whether it's an array or single value
+                  item.finalData[targetField] = isArray
+                    ? resolvedNewIds
+                    : resolvedNewIds[0];
+                  needsUpdate = true;
                 }
+              }
             }
+          }
+        }
 
-            // Update the importMap if changes were made to the item
-            if (needsUpdate) {
-                this.importMap.set(collectionKey, collectionData);
-                logger.info(`Updated item: ${JSON.stringify(item.finalData, undefined, 2)}`);
-            }
+        // Update the importMap if changes were made to the item
+        if (needsUpdate) {
+          this.importMap.set(collectionKey, collectionData);
+          logger.info(
+            `Updated item: ${JSON.stringify(item.finalData, undefined, 2)}`
+          );
         }
+      }
     }
-}
+  }
 
-findNewIdForOldId(oldId: string, idMapping: IdMapping, importDef: ImportDef) {
-  // First, check if this ID mapping is related to the users collection.
-  const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
-  const isUsersCollection = targetCollectionKey === this.getCollectionKey(this.config.usersCollectionName);
+  findNewIdForOldId(oldId: string, idMapping: IdMapping, importDef: ImportDef) {
+    // First, check if this ID mapping is related to the users collection.
+    const targetCollectionKey = this.getCollectionKey(
+      idMapping.targetCollection
+    );
+    const isUsersCollection =
+      targetCollectionKey ===
+      this.getCollectionKey(this.config.usersCollectionName);
 
-  // If handling users, check the mergedUserMap for any existing new ID.
-  if (isUsersCollection) {
+    // If handling users, check the mergedUserMap for any existing new ID.
+    if (isUsersCollection) {
       for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
-          if (oldIds.includes(oldId)) {
-              return newUserId;
-          }
+        if (oldIds.includes(oldId)) {
+          return newUserId;
+        }
       }
-  }
+    }
 
-  // If not a user or no merged ID found, check the regular ID mapping from old to new.
-  const targetCollectionData = this.importMap.get(targetCollectionKey);
-  if (targetCollectionData) {
-      const foundEntry = targetCollectionData.data.find(entry => entry.context[importDef.primaryKeyField] === oldId);
+    // If not a user or no merged ID found, check the regular ID mapping from old to new.
+    const targetCollectionData = this.importMap.get(targetCollectionKey);
+    if (targetCollectionData) {
+      const foundEntry = targetCollectionData.data.find(
+        (entry) => entry.context[importDef.primaryKeyField] === oldId
+      );
       if (foundEntry) {
-          return foundEntry.context.docId;  // Assuming `docId` stores the new ID after import
+        return foundEntry.context.docId; // Assuming `docId` stores the new ID after import
       }
-  }
-
-  logger.error(`No corresponding new ID found for ${oldId} in ${targetCollectionKey}`);
-  return null; // Return null if no new ID is found
-}
-
+    }
 
+    logger.error(
+      `No corresponding new ID found for ${oldId} in ${targetCollectionKey}`
+    );
+    return null; // Return null if no new ID is found
+  }
 
   private writeMapsToJsonFile() {
     const outputDir = path.resolve(process.cwd());

package/src/migrations/indexes.ts

@@ -14,13 +14,6 @@ export const createOrUpdateIndex = async (
   if (existingIndex.total > 0) {
     await db.deleteIndex(dbId, collectionId, existingIndex.indexes[0].key);
   }
-  if (index.type === "fulltext" && index.attributes.length > 1) {
-    throw new Error(
-      `Fulltext index can only be created on a single attribute. Index: ${index.key}`
-    );
-  } else if (index.type === "fulltext") {
-    index.attributes = [index.attributes[0]];
-  }
   const newIndex = await db.createIndex(
     dbId,
     collectionId,