appwrite-utils-cli 0.0.62 → 0.0.64

package/dist/migrations/dataLoader.d.ts

@@ -623,6 +623,16 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             relatedCollection: string;
             relationType: "oneToMany" | "manyToOne" | "oneToOne" | "manyToMany";
             twoWay: boolean;
+            /**
+             * Generates attribute mappings with post-import actions based on the provided attribute mappings.
+             * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
+             * and update the field with the file's ID if necessary.
+             *
+             * @param attributeMappings - The attribute mappings from the import definition.
+             * @param context - The context object containing information about the database, collection, and document.
+             * @param item - The item being imported, used for resolving template paths in fileData mappings.
+             * @returns The attribute mappings updated with any necessary post-import actions.
+             */
             twoWayKey: string;
             onDelete: "setNull" | "cascade" | "restrict";
             side: "parent" | "child";
@@ -1235,6 +1245,16 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             relatedCollection: string;
             relationType: "oneToMany" | "manyToOne" | "oneToOne" | "manyToMany";
             twoWay: boolean;
+            /**
+             * Generates attribute mappings with post-import actions based on the provided attribute mappings.
+             * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
+             * and update the field with the file's ID if necessary.
+             *
+             * @param attributeMappings - The attribute mappings from the import definition.
+             * @param context - The context object containing information about the database, collection, and document.
+             * @param item - The item being imported, used for resolving template paths in fileData mappings.
+             * @returns The attribute mappings updated with any necessary post-import actions.
+             */
             twoWayKey: string;
             onDelete: "setNull" | "cascade" | "restrict";
             side: "parent" | "child";
@@ -1635,6 +1655,16 @@ export declare class DataLoader {
                 relatedCollection: string;
                 relationType: "oneToMany" | "manyToOne" | "oneToOne" | "manyToMany";
                 twoWay: boolean;
+                /**
+                 * Generates attribute mappings with post-import actions based on the provided attribute mappings.
+                 * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
+                 * and update the field with the file's ID if necessary.
+                 *
+                 * @param attributeMappings - The attribute mappings from the import definition.
+                 * @param context - The context object containing information about the database, collection, and document.
+                 * @param item - The item being imported, used for resolving template paths in fileData mappings.
+                 * @returns The attribute mappings updated with any necessary post-import actions.
+                 */
                 twoWayKey: string;
                 onDelete: "setNull" | "cascade" | "restrict";
                 side: "parent" | "child";

package/dist/migrations/dataLoader.js

@@ -278,6 +278,11 @@ export class DataLoader {
                     userId: user.$id,
                     docId: user.$id,
                 },
+                context: {
+                    ...user,
+                    userId: user.$id,
+                    docId: user.$id,
+                },
                 rawData: user,
             });
             this.importMap.set(this.getCollectionKey("users"), importData);
@@ -291,7 +296,8 @@ export class DataLoader {
         console.log("---------------------------------");
         await this.setupMaps(dbId);
         const allUsers = await this.getAllUsers();
-        console.log(`Fetched ${allUsers.length} users`);
+        console.log(`Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`);
+        await new Promise((resolve) => setTimeout(resolve, 5000));
         // Iterate over the configured databases to find the matching one
         for (const db of this.config.databases) {
             if (db.$id !== dbId) {
@@ -448,13 +454,6 @@ export class DataLoader {
                                         targetValue !== undefined &&
                                         targetValue !== null);
                                 });
-                                // Log and skip if no matching data found
-                                if (!foundData.length) {
-                                    console.log(`No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`);
-                                    logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
-                                    continue;
-                                }
-                                needsUpdate = true;
                                 const getCurrentDataFiltered = (currentData) => {
                                     if (Array.isArray(currentData.finalData[fieldToSetKey])) {
                                         return currentData.finalData[fieldToSetKey].filter((data) => `${data}` !== `${valueToMatch}`);
@@ -463,6 +462,13 @@ export class DataLoader {
                                 };
                                 // Get the current data to be updated
                                 const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
+                                // Log and skip if no matching data found
+                                if (!foundData.length) {
+                                    console.log(`No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`);
+                                    logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
+                                    continue;
+                                }
+                                needsUpdate = true;
                                 // Extract the new data to set
                                 const newData = foundData.map((data) => {
                                     const valueFound = this.getValueFromData(data.finalData, data.context, idMapping.targetField);
@@ -575,6 +581,11 @@ export class DataLoader {
      * @returns The transformed item with user-specific keys removed.
      */
     prepareUserData(item, attributeMappings, primaryKeyField, newId) {
+        if (this.userExistsMap.has(newId) ||
+            Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+            Array.from(this.phoneToUserIdMap.values()).includes(newId)) {
+            newId = this.getTrueUniqueId(this.getCollectionKey("users"));
+        }
         let transformedItem = this.transformData(item, attributeMappings);
         const userData = AuthUserCreateSchema.safeParse(transformedItem);
         if (!userData.success || !(userData.data.email && userData.data.phone)) {
@@ -728,9 +739,18 @@ export class DataLoader {
             let { transformedItem, existingId, userData } = this.prepareUserData(item, importDef.attributeMappings, importDef.primaryKeyField, this.getTrueUniqueId(this.getCollectionKey("users")));
             logger.info(`In create user -- transformedItem: ${JSON.stringify(transformedItem, null, 2)}`);
             // Generate a new unique ID for the item or use existing ID
-            if (!existingId) {
+            if (!existingId && !userData.finalData?.userId) {
                 // No existing user ID, generate a new unique ID
-                existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
+                existingId = this.getTrueUniqueId(this.getCollectionKey(collection.name));
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
+            }
+            else if (!existingId && userData.finalData?.userId) {
+                // Existing user ID, use it as the new ID
+                existingId = userData.finalData.userId;
                 transformedItem = {
                     ...transformedItem,
                     userId: existingId,
@@ -751,7 +771,9 @@ export class DataLoader {
                     // Found a duplicate oldId, now decide how to merge or handle these duplicates
                     for (const data of currentData.data) {
                         if (data.finalData.docId === oldId ||
-                            data.finalData.userId === oldId) {
+                            data.finalData.userId === oldId ||
+                            data.context.docId === oldId ||
+                            data.context.userId === oldId) {
                             transformedItem = this.mergeObjects(data.finalData, transformedItem);
                         }
                     }
@@ -787,30 +809,41 @@ export class DataLoader {
                 ...importDef,
                 attributeMappings: mappingsWithActions,
             };
+            const updatedData = this.importMap.get(this.getCollectionKey(collection.name));
             let foundData = false;
-            for (let i = 0; i < currentData.data.length; i++) {
-                if (currentData.data[i].finalData.docId === existingId ||
-                    currentData.data[i].finalData.userId === existingId) {
-                    currentData.data[i].finalData = this.mergeObjects(currentData.data[i].finalData, transformedItem);
-                    currentData.data[i].context = {
-                        ...currentData.data[i].context,
-                        ...context,
+            for (let i = 0; i < updatedData.data.length; i++) {
+                if (updatedData.data[i].finalData.docId === existingId ||
+                    updatedData.data[i].finalData.userId === existingId ||
+                    updatedData.data[i].context.docId === existingId ||
+                    updatedData.data[i].context.userId === existingId) {
+                    updatedData.data[i].finalData = this.mergeObjects(updatedData.data[i].finalData, transformedItem);
+                    updatedData.data[i].context = this.mergeObjects(updatedData.data[i].context, context);
+                    const mergedImportDef = {
+                        ...updatedData.data[i].importDef,
+                        idMappings: [
+                            ...(updatedData.data[i].importDef?.idMappings || []),
+                            ...(newImportDef.idMappings || []),
+                        ],
+                        attributeMappings: [
+                            ...(updatedData.data[i].importDef?.attributeMappings || []),
+                            ...(newImportDef.attributeMappings || []),
+                        ],
                     };
-                    currentData.data[i].importDef = newImportDef;
-                    this.importMap.set(this.getCollectionKey(collection.name), currentData);
+                    updatedData.data[i].importDef = mergedImportDef;
+                    this.importMap.set(this.getCollectionKey(collection.name), updatedData);
                     this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
                     foundData = true;
                 }
             }
             if (!foundData) {
                 // Add new data to the associated collection
-                currentData.data.push({
+                updatedData.data.push({
                     rawData: item,
                     context: context,
                     importDef: newImportDef,
                     finalData: transformedItem,
                 });
-                this.importMap.set(this.getCollectionKey(collection.name), currentData);
+                this.importMap.set(this.getCollectionKey(collection.name), updatedData);
                 this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
             }
         }
@@ -854,13 +887,7 @@ export class DataLoader {
             // Create a context object for the item, including the new ID
             let context = this.createContext(db, collection, item, itemIdNew);
             // Transform the item data based on the attribute mappings
-            const transformedData = this.transformData(item, importDef.attributeMappings);
-            if (collection.name.toLowerCase() === "councils") {
-                console.log("Transformed Council: ", transformedData);
-            }
-            if (isRegions) {
-                logger.info(`Transformed region: ${JSON.stringify(transformedData, null, 2)}`);
-            }
+            let transformedData = this.transformData(item, importDef.attributeMappings);
             // If a primary key field is defined, handle the ID mapping and check for duplicates
             if (importDef.primaryKeyField) {
                 const oldId = item[importDef.primaryKeyField];

package/dist/migrations/importController.js

@@ -157,6 +157,7 @@ export class ImportController {
                 status: "in_progress",
             });
             const collectionData = dataLoader.importMap.get(dataLoader.getCollectionKey(collection.name));
+            console.log(`Processing collection: ${collection.name}...`);
             if (!collectionData) {
                 console.log("No collection data for ", collection.name);
                 continue;

package/dist/migrations/storage.js

@@ -253,7 +253,7 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
     let numberOfFiles = 0;
     if (fromFiles.files.length < 100) {
         for (const file of allFromFiles) {
-            const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+            const fileData = await tryAwaitWithRetry(async () => await storage.getFileDownload(file.bucketId, file.$id));
             const fileToCreate = InputFile.fromBuffer(Buffer.from(fileData), file.name);
             console.log(`Creating file: ${file.name}`);
             tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
@@ -263,10 +263,10 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
     else {
         lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
         while (lastFileId) {
-            const files = await storage.listFiles(fromBucketId, [
+            const files = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [
                 Query.limit(100),
                 Query.cursorAfter(lastFileId),
-            ]);
+            ]));
             allFromFiles.push(...files.files);
             if (files.files.length < 100) {
                 lastFileId = undefined;
@@ -276,7 +276,7 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
             }
         }
         for (const file of allFromFiles) {
-            const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+            const fileData = await tryAwaitWithRetry(async () => await storage.getFileDownload(file.bucketId, file.$id));
             const fileToCreate = InputFile.fromBuffer(Buffer.from(fileData), file.name);
             await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
             numberOfFiles++;
@@ -295,10 +295,10 @@ export const transferStorageLocalToRemote = async (localStorage, endpoint, proje
     if (fromFiles.files.length === 100) {
         lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
         while (lastFileId) {
-            const files = await localStorage.listFiles(fromBucketId, [
+            const files = await tryAwaitWithRetry(async () => await localStorage.listFiles(fromBucketId, [
                 Query.limit(100),
                 Query.cursorAfter(lastFileId),
-            ]);
+            ]));
             allFromFiles.push(...files.files);
             if (files.files.length < 100) {
                 break;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.62",
+  "version": "0.0.64",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/dataLoader.ts

@@ -348,6 +348,11 @@ export class DataLoader {
           userId: user.$id,
           docId: user.$id,
         },
+        context: {
+          ...user,
+          userId: user.$id,
+          docId: user.$id,
+        },
         rawData: user,
       });
       this.importMap.set(this.getCollectionKey("users"), importData);
@@ -362,7 +367,10 @@ export class DataLoader {
     console.log("---------------------------------");
     await this.setupMaps(dbId);
     const allUsers = await this.getAllUsers();
-    console.log(`Fetched ${allUsers.length} users`);
+    console.log(
+      `Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`
+    );
+    await new Promise((resolve) => setTimeout(resolve, 5000));
     // Iterate over the configured databases to find the matching one
     for (const db of this.config.databases) {
       if (db.$id !== dbId) {
@@ -568,6 +576,20 @@ export class DataLoader {
                     );
                   }
                 );
+
+                const getCurrentDataFiltered = (currentData: any) => {
+                  if (Array.isArray(currentData.finalData[fieldToSetKey])) {
+                    return currentData.finalData[fieldToSetKey].filter(
+                      (data: any) => `${data}` !== `${valueToMatch}`
+                    );
+                  }
+                  return currentData.finalData[fieldToSetKey];
+                };
+
+                // Get the current data to be updated
+                const currentDataFiltered = getCurrentDataFiltered(
+                  collectionData.data[i]
+                );
                 // Log and skip if no matching data found
                 if (!foundData.length) {
                   console.log(
@@ -585,20 +607,6 @@ export class DataLoader {
 
                 needsUpdate = true;
 
-                const getCurrentDataFiltered = (currentData: any) => {
-                  if (Array.isArray(currentData.finalData[fieldToSetKey])) {
-                    return currentData.finalData[fieldToSetKey].filter(
-                      (data: any) => `${data}` !== `${valueToMatch}`
-                    );
-                  }
-                  return currentData.finalData[fieldToSetKey];
-                };
-
-                // Get the current data to be updated
-                const currentDataFiltered = getCurrentDataFiltered(
-                  collectionData.data[i]
-                );
-
                 // Extract the new data to set
                 const newData = foundData.map((data) => {
                   const valueFound = this.getValueFromData(
@@ -752,6 +760,13 @@ export class DataLoader {
       finalData: z.infer<typeof AuthUserCreateSchema>;
     };
   } {
+    if (
+      this.userExistsMap.has(newId) ||
+      Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+      Array.from(this.phoneToUserIdMap.values()).includes(newId)
+    ) {
+      newId = this.getTrueUniqueId(this.getCollectionKey("users"));
+    }
     let transformedItem = this.transformData(item, attributeMappings);
     const userData = AuthUserCreateSchema.safeParse(transformedItem);
     if (!userData.success || !(userData.data.email && userData.data.phone)) {
@@ -940,9 +955,19 @@ export class DataLoader {
       );
 
       // Generate a new unique ID for the item or use existing ID
-      if (!existingId) {
+      if (!existingId && !userData.finalData?.userId) {
         // No existing user ID, generate a new unique ID
-        existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
+        existingId = this.getTrueUniqueId(
+          this.getCollectionKey(collection.name)
+        );
+        transformedItem = {
+          ...transformedItem,
+          userId: existingId,
+          docId: existingId,
+        };
+      } else if (!existingId && userData.finalData?.userId) {
+        // Existing user ID, use it as the new ID
+        existingId = userData.finalData.userId;
         transformedItem = {
           ...transformedItem,
           userId: existingId,
@@ -951,7 +976,7 @@ export class DataLoader {
       }
 
       // Create a context object for the item, including the new ID
-      let context = this.createContext(db, collection, item, existingId);
+      let context = this.createContext(db, collection, item, existingId!);
 
       // Merge the transformed data into the context
       context = { ...context, ...transformedItem, ...userData.finalData };
@@ -970,7 +995,9 @@ export class DataLoader {
           for (const data of currentData.data) {
             if (
               data.finalData.docId === oldId ||
-              data.finalData.userId === oldId
+              data.finalData.userId === oldId ||
+              data.context.docId === oldId ||
+              data.context.userId === oldId
             ) {
               transformedItem = this.mergeObjects(
                 data.finalData,
@@ -1022,41 +1049,59 @@ export class DataLoader {
         attributeMappings: mappingsWithActions,
       };
 
+      const updatedData = this.importMap.get(
+        this.getCollectionKey(collection.name)
+      )!;
+
       let foundData = false;
-      for (let i = 0; i < currentData.data.length; i++) {
+      for (let i = 0; i < updatedData.data.length; i++) {
         if (
-          currentData.data[i].finalData.docId === existingId ||
-          currentData.data[i].finalData.userId === existingId
+          updatedData.data[i].finalData.docId === existingId ||
+          updatedData.data[i].finalData.userId === existingId ||
+          updatedData.data[i].context.docId === existingId ||
+          updatedData.data[i].context.userId === existingId
         ) {
-          currentData.data[i].finalData = this.mergeObjects(
-            currentData.data[i].finalData,
+          updatedData.data[i].finalData = this.mergeObjects(
+            updatedData.data[i].finalData,
             transformedItem
           );
-          currentData.data[i].context = {
-            ...currentData.data[i].context,
-            ...context,
+          updatedData.data[i].context = this.mergeObjects(
+            updatedData.data[i].context,
+            context
+          );
+          const mergedImportDef = {
+            ...updatedData.data[i].importDef,
+            idMappings: [
+              ...(updatedData.data[i].importDef?.idMappings || []),
+              ...(newImportDef.idMappings || []),
+            ],
+            attributeMappings: [
+              ...(updatedData.data[i].importDef?.attributeMappings || []),
+              ...(newImportDef.attributeMappings || []),
+            ],
           };
-          currentData.data[i].importDef = newImportDef;
+          updatedData.data[i].importDef = mergedImportDef as ImportDef;
           this.importMap.set(
             this.getCollectionKey(collection.name),
-            currentData
+            updatedData
           );
           this.oldIdToNewIdPerCollectionMap.set(
             this.getCollectionKey(collection.name),
             collectionOldIdToNewIdMap!
           );
+
           foundData = true;
         }
       }
       if (!foundData) {
         // Add new data to the associated collection
-        currentData.data.push({
+        updatedData.data.push({
           rawData: item,
           context: context,
           importDef: newImportDef,
           finalData: transformedItem,
         });
-        this.importMap.set(this.getCollectionKey(collection.name), currentData);
+        this.importMap.set(this.getCollectionKey(collection.name), updatedData);
         this.oldIdToNewIdPerCollectionMap.set(
           this.getCollectionKey(collection.name),
           collectionOldIdToNewIdMap!
@@ -1119,18 +1164,10 @@ export class DataLoader {
       // Create a context object for the item, including the new ID
       let context = this.createContext(db, collection, item, itemIdNew);
       // Transform the item data based on the attribute mappings
-      const transformedData = this.transformData(
+      let transformedData = this.transformData(
         item,
         importDef.attributeMappings
       );
-      if (collection.name.toLowerCase() === "councils") {
-        console.log("Transformed Council: ", transformedData);
-      }
-      if (isRegions) {
-        logger.info(
-          `Transformed region: ${JSON.stringify(transformedData, null, 2)}`
-        );
-      }
       // If a primary key field is defined, handle the ID mapping and check for duplicates
       if (importDef.primaryKeyField) {
         const oldId = item[importDef.primaryKeyField];

package/src/migrations/importController.ts

@@ -237,6 +237,7 @@ export class ImportController {
       const collectionData = dataLoader.importMap.get(
         dataLoader.getCollectionKey(collection.name)
       );
+      console.log(`Processing collection: ${collection.name}...`);
       if (!collectionData) {
         console.log("No collection data for ", collection.name);
         continue;

package/src/migrations/storage.ts

@@ -390,7 +390,9 @@ export const transferStorageLocalToLocal = async (
   let numberOfFiles = 0;
   if (fromFiles.files.length < 100) {
     for (const file of allFromFiles) {
-      const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+      const fileData = await tryAwaitWithRetry(
+        async () => await storage.getFileDownload(file.bucketId, file.$id)
+      );
       const fileToCreate = InputFile.fromBuffer(
         Buffer.from(fileData),
         file.name
@@ -410,10 +412,13 @@ export const transferStorageLocalToLocal = async (
   } else {
     lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
     while (lastFileId) {
-      const files = await storage.listFiles(fromBucketId, [
-        Query.limit(100),
-        Query.cursorAfter(lastFileId),
-      ]);
+      const files = await tryAwaitWithRetry(
+        async () =>
+          await storage.listFiles(fromBucketId, [
+            Query.limit(100),
+            Query.cursorAfter(lastFileId!),
+          ])
+      );
       allFromFiles.push(...files.files);
       if (files.files.length < 100) {
         lastFileId = undefined;
@@ -422,7 +427,9 @@ export const transferStorageLocalToLocal = async (
       }
     }
     for (const file of allFromFiles) {
-      const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+      const fileData = await tryAwaitWithRetry(
+        async () => await storage.getFileDownload(file.bucketId, file.$id)
+      );
       const fileToCreate = InputFile.fromBuffer(
         Buffer.from(fileData),
         file.name
@@ -467,10 +474,13 @@ export const transferStorageLocalToRemote = async (
   if (fromFiles.files.length === 100) {
     lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
     while (lastFileId) {
-      const files = await localStorage.listFiles(fromBucketId, [
-        Query.limit(100),
-        Query.cursorAfter(lastFileId),
-      ]);
+      const files = await tryAwaitWithRetry(
+        async () =>
+          await localStorage.listFiles(fromBucketId, [
+            Query.limit(100),
+            Query.cursorAfter(lastFileId!),
+          ])
+      );
       allFromFiles.push(...files.files);
       if (files.files.length < 100) {
         break;