appwrite-utils-cli 0.0.61 → 0.0.63

package/dist/migrations/afterImportActions.js

@@ -26,7 +26,7 @@ export const afterImportActions = {
     updateCreatedDocument: async (config, dbId, collId, docId, data) => {
         try {
             const db = getDatabaseFromConfig(config);
-            await db.updateDocument(dbId, collId, docId, data);
+            await tryAwaitWithRetry(async () => await db.updateDocument(dbId, collId, docId, data));
         }
         catch (error) {
             console.error("Error updating document: ", error);
@@ -35,11 +35,11 @@ export const afterImportActions = {
     checkAndUpdateFieldInDocument: async (config, dbId, collId, docId, fieldName, oldFieldValue, newFieldValue) => {
         try {
             const db = getDatabaseFromConfig(config);
-            const doc = await db.getDocument(dbId, collId, docId);
+            const doc = await tryAwaitWithRetry(async () => await db.getDocument(dbId, collId, docId));
             if (doc[fieldName] == oldFieldValue) {
-                await db.updateDocument(dbId, collId, docId, {
+                await tryAwaitWithRetry(async () => await db.updateDocument(dbId, collId, docId, {
                     [fieldName]: newFieldValue,
-                });
+                }));
             }
         }
         catch (error) {
@@ -50,10 +50,10 @@ export const afterImportActions = {
         const db = getDatabaseFromConfig(config);
         // Helper function to find a collection ID by name or return the ID if given
         const findCollectionId = async (collectionIdentifier) => {
-            const collectionsPulled = await db.listCollections(dbId, [
+            const collectionsPulled = await tryAwaitWithRetry(async () => await db.listCollections(dbId, [
                 Query.limit(25),
                 Query.equal("name", collectionIdentifier),
-            ]);
+            ]));
             if (collectionsPulled.total > 0) {
                 return collectionsPulled.collections[0].$id;
             }
@@ -71,9 +71,9 @@ export const afterImportActions = {
             const valueToSet = otherDoc[otherFieldName];
             if (valueToSet) {
                 // Update the target document
-                await db.updateDocument(dbId, targetCollectionId, docId, {
+                await tryAwaitWithRetry(async () => await db.updateDocument(dbId, targetCollectionId, docId, {
                     [fieldName]: valueToSet,
-                });
+                }));
             }
             console.log(`Field ${fieldName} updated successfully in document ${docId}.`);
         }
@@ -89,17 +89,17 @@ export const afterImportActions = {
         const db = getDatabaseFromConfig(config);
         // Helper function to find a collection ID by name or return the ID if given
         const findCollectionId = async (collectionIdentifier) => {
-            const collections = await db.listCollections(dbId, [
+            const collections = await tryAwaitWithRetry(async () => await db.listCollections(dbId, [
                 Query.equal("name", collectionIdentifier),
                 Query.limit(1),
-            ]);
+            ]));
             return collections.total > 0
                 ? collections.collections[0].$id
                 : collectionIdentifier;
         };
         // Function to check if the target field is an array
         const isTargetFieldArray = async (collectionId, fieldName) => {
-            const collection = await db.getCollection(dbId, collectionId);
+            const collection = await tryAwaitWithRetry(async () => await db.getCollection(dbId, collectionId));
             const attribute = collection.attributes.find((attr) => attr.key === fieldName);
             // @ts-ignore
             return attribute?.array === true;
@@ -119,7 +119,7 @@ export const afterImportActions = {
                     queries.push(Query.cursorAfter(cursor));
                 }
                 queries.push(Query.limit(docLimit));
-                const response = await db.listDocuments(dbId, otherCollectionId, queries);
+                const response = await tryAwaitWithRetry(async () => await db.listDocuments(dbId, otherCollectionId, queries));
                 const documents = response.documents;
                 if (documents.length === 0 || documents.length < docLimit) {
                     return documents;
@@ -134,7 +134,7 @@ export const afterImportActions = {
                 const updatePayload = targetFieldIsArray
                     ? { [fieldName]: documentIds }
                     : { [fieldName]: documentIds[0] };
-                await db.updateDocument(dbId, targetCollectionId, docId, updatePayload);
+                await tryAwaitWithRetry(async () => await db.updateDocument(dbId, targetCollectionId, docId, updatePayload));
                 console.log(`Field ${fieldName} updated successfully in document ${docId} with ${documentIds.length} document IDs.`);
             }
         }
@@ -145,10 +145,10 @@ export const afterImportActions = {
     setTargetFieldFromOtherCollectionDocumentsByMatchingField: async (config, dbId, collIdOrName, docId, fieldName, otherCollIdOrName, matchingFieldName, matchingFieldValue, targetField) => {
         const db = getDatabaseFromConfig(config);
         const findCollectionId = async (collectionIdentifier) => {
-            const collections = await db.listCollections(dbId, [
+            const collections = await tryAwaitWithRetry(async () => await db.listCollections(dbId, [
                 Query.equal("name", collectionIdentifier),
                 Query.limit(1),
-            ]);
+            ]));
             return collections.total > 0
                 ? collections.collections[0].$id
                 : collectionIdentifier;
@@ -172,7 +172,7 @@ export const afterImportActions = {
                 if (cursor) {
                     queries.push(Query.cursorAfter(cursor));
                 }
-                const response = await db.listDocuments(dbId, otherCollectionId, queries);
+                const response = await tryAwaitWithRetry(async () => await db.listDocuments(dbId, otherCollectionId, queries));
                 const documents = response.documents;
                 if (documents.length === 0 || documents.length < docLimit) {
                     return documents;
@@ -188,7 +188,7 @@ export const afterImportActions = {
                 const updatePayload = targetFieldIsArray
                     ? { [fieldName]: targetFieldValues }
                     : { [fieldName]: targetFieldValues[0] };
-                await db.updateDocument(dbId, targetCollectionId, docId, updatePayload);
+                await tryAwaitWithRetry(async () => await db.updateDocument(dbId, targetCollectionId, docId, updatePayload));
                 console.log(`Field ${fieldName} updated successfully in document ${docId} with values from field ${targetField}.`);
             }
         }
@@ -199,22 +199,20 @@ export const afterImportActions = {
     createOrGetBucket: async (config, bucketName, bucketId, permissions, fileSecurity, enabled, maxFileSize, allowedExtensions, compression, encryption, antivirus) => {
         try {
             const storage = getStorageFromConfig(config);
-            const bucket = await storage.listBuckets([
-                Query.equal("name", bucketName),
-            ]);
+            const bucket = await tryAwaitWithRetry(async () => await storage.listBuckets([Query.equal("name", bucketName)]));
             if (bucket.buckets.length > 0) {
                 return bucket.buckets[0];
             }
             else if (bucketId) {
                 try {
-                    return await storage.getBucket(bucketId);
+                    return await tryAwaitWithRetry(async () => await storage.getBucket(bucketId));
                 }
                 catch (error) {
-                    return await storage.createBucket(bucketId, bucketName, permissions, fileSecurity, enabled, maxFileSize, allowedExtensions, compression, encryption, antivirus);
+                    return await tryAwaitWithRetry(async () => await storage.createBucket(bucketId, bucketName, permissions, fileSecurity, enabled, maxFileSize, allowedExtensions, compression, encryption, antivirus));
                 }
             }
             else {
-                return await storage.createBucket(bucketId || ID.unique(), bucketName, permissions, fileSecurity, enabled, maxFileSize, allowedExtensions, compression, encryption, antivirus);
+                return await tryAwaitWithRetry(async () => await storage.createBucket(bucketId || ID.unique(), bucketName, permissions, fileSecurity, enabled, maxFileSize, allowedExtensions, compression, encryption, antivirus));
             }
         }
         catch (error) {
@@ -225,7 +223,7 @@ export const afterImportActions = {
         try {
             const db = getDatabaseFromConfig(config);
             const storage = getStorageFromConfig(config);
-            const collection = await db.getCollection(dbId, collId);
+            const collection = await tryAwaitWithRetry(async () => await db.getCollection(dbId, collId));
             const attributes = collection.attributes;
             const attribute = attributes.find((a) => a.key === fieldName);
             // console.log(
@@ -255,7 +253,7 @@ export const afterImportActions = {
                 const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "appwrite_tmp"));
                 const tempFilePath = path.join(tempDir, fileName);
                 // Download the file using fetch
-                const response = await fetch(filePath);
+                const response = await tryAwaitWithRetry(async () => await fetch(filePath));
                 if (!response.ok)
                     console.error(`Failed to fetch ${filePath}: ${response.statusText} for document ${docId} with field ${fieldName}`);
                 // Use arrayBuffer if buffer is not available

package/dist/migrations/dataLoader.js

@@ -291,7 +291,8 @@ export class DataLoader {
         console.log("---------------------------------");
         await this.setupMaps(dbId);
         const allUsers = await this.getAllUsers();
-        console.log(`Fetched ${allUsers.length} users`);
+        console.log(`Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`);
+        await new Promise((resolve) => setTimeout(resolve, 5000));
         // Iterate over the configured databases to find the matching one
         for (const db of this.config.databases) {
             if (db.$id !== dbId) {
@@ -728,9 +729,18 @@ export class DataLoader {
             let { transformedItem, existingId, userData } = this.prepareUserData(item, importDef.attributeMappings, importDef.primaryKeyField, this.getTrueUniqueId(this.getCollectionKey("users")));
             logger.info(`In create user -- transformedItem: ${JSON.stringify(transformedItem, null, 2)}`);
             // Generate a new unique ID for the item or use existing ID
-            if (!existingId) {
+            if (!existingId && !userData.finalData?.userId) {
                 // No existing user ID, generate a new unique ID
-                existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
+                existingId = this.getTrueUniqueId(this.getCollectionKey(collection.name));
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
+            }
+            else if (!existingId && userData.finalData?.userId) {
+                // Existing user ID, use it as the new ID
+                existingId = userData.finalData.userId;
                 transformedItem = {
                     ...transformedItem,
                     userId: existingId,
@@ -751,7 +761,9 @@ export class DataLoader {
                     // Found a duplicate oldId, now decide how to merge or handle these duplicates
                     for (const data of currentData.data) {
                         if (data.finalData.docId === oldId ||
-                            data.finalData.userId === oldId) {
+                            data.finalData.userId === oldId ||
+                            data.context.docId === oldId ||
+                            data.context.userId === oldId) {
                             transformedItem = this.mergeObjects(data.finalData, transformedItem);
                         }
                     }
@@ -787,30 +799,30 @@ export class DataLoader {
                 ...importDef,
                 attributeMappings: mappingsWithActions,
             };
+            const updatedData = this.importMap.get(this.getCollectionKey(collection.name));
             let foundData = false;
-            for (let i = 0; i < currentData.data.length; i++) {
-                if (currentData.data[i].finalData.docId === existingId ||
-                    currentData.data[i].finalData.userId === existingId) {
-                    currentData.data[i].finalData = this.mergeObjects(currentData.data[i].finalData, transformedItem);
-                    currentData.data[i].context = {
-                        ...currentData.data[i].context,
-                        ...context,
-                    };
-                    currentData.data[i].importDef = newImportDef;
-                    this.importMap.set(this.getCollectionKey(collection.name), currentData);
+            for (let i = 0; i < updatedData.data.length; i++) {
+                if (updatedData.data[i].finalData.docId === existingId ||
+                    updatedData.data[i].finalData.userId === existingId ||
+                    updatedData.data[i].context.docId === existingId ||
+                    updatedData.data[i].context.userId === existingId) {
+                    updatedData.data[i].finalData = this.mergeObjects(updatedData.data[i].finalData, transformedItem);
+                    updatedData.data[i].context = this.mergeObjects(updatedData.data[i].context, context);
+                    updatedData.data[i].importDef = this.mergeObjects(updatedData.data[i].importDef, newImportDef);
+                    this.importMap.set(this.getCollectionKey(collection.name), updatedData);
                     this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
                     foundData = true;
                 }
             }
             if (!foundData) {
                 // Add new data to the associated collection
-                currentData.data.push({
+                updatedData.data.push({
                     rawData: item,
                     context: context,
                     importDef: newImportDef,
                     finalData: transformedItem,
                 });
-                this.importMap.set(this.getCollectionKey(collection.name), currentData);
+                this.importMap.set(this.getCollectionKey(collection.name), updatedData);
                 this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
             }
         }
@@ -854,13 +866,7 @@ export class DataLoader {
             // Create a context object for the item, including the new ID
             let context = this.createContext(db, collection, item, itemIdNew);
             // Transform the item data based on the attribute mappings
-            const transformedData = this.transformData(item, importDef.attributeMappings);
-            if (collection.name.toLowerCase() === "councils") {
-                console.log("Transformed Council: ", transformedData);
-            }
-            if (isRegions) {
-                logger.info(`Transformed region: ${JSON.stringify(transformedData, null, 2)}`);
-            }
+            let transformedData = this.transformData(item, importDef.attributeMappings);
             // If a primary key field is defined, handle the ID mapping and check for duplicates
             if (importDef.primaryKeyField) {
                 const oldId = item[importDef.primaryKeyField];

package/dist/migrations/importController.js

@@ -157,6 +157,7 @@ export class ImportController {
                 status: "in_progress",
             });
             const collectionData = dataLoader.importMap.get(dataLoader.getCollectionKey(collection.name));
+            console.log(`Processing collection: ${collection.name}...`);
             if (!collectionData) {
                 console.log("No collection data for ", collection.name);
                 continue;

package/dist/migrations/storage.js

@@ -253,7 +253,7 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
     let numberOfFiles = 0;
     if (fromFiles.files.length < 100) {
         for (const file of allFromFiles) {
-            const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+            const fileData = await tryAwaitWithRetry(async () => await storage.getFileDownload(file.bucketId, file.$id));
             const fileToCreate = InputFile.fromBuffer(Buffer.from(fileData), file.name);
             console.log(`Creating file: ${file.name}`);
             tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
@@ -263,10 +263,10 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
     else {
         lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
         while (lastFileId) {
-            const files = await storage.listFiles(fromBucketId, [
+            const files = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [
                 Query.limit(100),
                 Query.cursorAfter(lastFileId),
-            ]);
+            ]));
             allFromFiles.push(...files.files);
             if (files.files.length < 100) {
                 lastFileId = undefined;
@@ -276,7 +276,7 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
             }
         }
         for (const file of allFromFiles) {
-            const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+            const fileData = await tryAwaitWithRetry(async () => await storage.getFileDownload(file.bucketId, file.$id));
             const fileToCreate = InputFile.fromBuffer(Buffer.from(fileData), file.name);
             await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
             numberOfFiles++;
@@ -295,10 +295,10 @@ export const transferStorageLocalToRemote = async (localStorage, endpoint, proje
     if (fromFiles.files.length === 100) {
         lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
         while (lastFileId) {
-            const files = await localStorage.listFiles(fromBucketId, [
+            const files = await tryAwaitWithRetry(async () => await localStorage.listFiles(fromBucketId, [
                 Query.limit(100),
                 Query.cursorAfter(lastFileId),
-            ]);
+            ]));
             allFromFiles.push(...files.files);
             if (files.files.length < 100) {
                 break;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.61",
+  "version": "0.0.63",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/afterImportActions.ts

@@ -47,7 +47,9 @@ export const afterImportActions = {
   ) => {
     try {
       const db = getDatabaseFromConfig(config);
-      await db.updateDocument(dbId, collId, docId, data);
+      await tryAwaitWithRetry(
+        async () => await db.updateDocument(dbId, collId, docId, data)
+      );
     } catch (error) {
       console.error("Error updating document: ", error);
     }
@@ -63,11 +65,16 @@ export const afterImportActions = {
   ) => {
     try {
       const db = getDatabaseFromConfig(config);
-      const doc = await db.getDocument(dbId, collId, docId);
+      const doc = await tryAwaitWithRetry(
+        async () => await db.getDocument(dbId, collId, docId)
+      );
       if (doc[fieldName as keyof typeof doc] == oldFieldValue) {
-        await db.updateDocument(dbId, collId, docId, {
-          [fieldName]: newFieldValue,
-        });
+        await tryAwaitWithRetry(
+          async () =>
+            await db.updateDocument(dbId, collId, docId, {
+              [fieldName]: newFieldValue,
+            })
+        );
       }
     } catch (error) {
       console.error("Error updating document: ", error);
@@ -87,10 +94,13 @@ export const afterImportActions = {
 
     // Helper function to find a collection ID by name or return the ID if given
     const findCollectionId = async (collectionIdentifier: string) => {
-      const collectionsPulled = await db.listCollections(dbId, [
-        Query.limit(25),
-        Query.equal("name", collectionIdentifier),
-      ]);
+      const collectionsPulled = await tryAwaitWithRetry(
+        async () =>
+          await db.listCollections(dbId, [
+            Query.limit(25),
+            Query.equal("name", collectionIdentifier),
+          ])
+      );
       if (collectionsPulled.total > 0) {
         return collectionsPulled.collections[0].$id;
       } else {
@@ -114,9 +124,12 @@ export const afterImportActions = {
 
       if (valueToSet) {
         // Update the target document
-        await db.updateDocument(dbId, targetCollectionId, docId, {
-          [fieldName]: valueToSet,
-        });
+        await tryAwaitWithRetry(
+          async () =>
+            await db.updateDocument(dbId, targetCollectionId, docId, {
+              [fieldName]: valueToSet,
+            })
+        );
       }
 
       console.log(
@@ -148,10 +161,13 @@ export const afterImportActions = {
 
     // Helper function to find a collection ID by name or return the ID if given
     const findCollectionId = async (collectionIdentifier: string) => {
-      const collections = await db.listCollections(dbId, [
-        Query.equal("name", collectionIdentifier),
-        Query.limit(1),
-      ]);
+      const collections = await tryAwaitWithRetry(
+        async () =>
+          await db.listCollections(dbId, [
+            Query.equal("name", collectionIdentifier),
+            Query.limit(1),
+          ])
+      );
       return collections.total > 0
         ? collections.collections[0].$id
         : collectionIdentifier;
@@ -162,7 +178,9 @@ export const afterImportActions = {
       collectionId: string,
       fieldName: string
     ) => {
-      const collection = await db.getCollection(dbId, collectionId);
+      const collection = await tryAwaitWithRetry(
+        async () => await db.getCollection(dbId, collectionId)
+      );
       const attribute = collection.attributes.find(
         (attr: any) => attr.key === fieldName
       );
@@ -191,10 +209,8 @@ export const afterImportActions = {
           queries.push(Query.cursorAfter(cursor));
         }
         queries.push(Query.limit(docLimit));
-        const response = await db.listDocuments(
-          dbId,
-          otherCollectionId,
-          queries
+        const response = await tryAwaitWithRetry(
+          async () => await db.listDocuments(dbId, otherCollectionId, queries)
         );
         const documents = response.documents;
         if (documents.length === 0 || documents.length < docLimit) {
@@ -212,7 +228,15 @@ export const afterImportActions = {
         const updatePayload = targetFieldIsArray
           ? { [fieldName]: documentIds }
           : { [fieldName]: documentIds[0] };
-        await db.updateDocument(dbId, targetCollectionId, docId, updatePayload);
+        await tryAwaitWithRetry(
+          async () =>
+            await db.updateDocument(
+              dbId,
+              targetCollectionId,
+              docId,
+              updatePayload
+            )
+        );
 
         console.log(
           `Field ${fieldName} updated successfully in document ${docId} with ${documentIds.length} document IDs.`
@@ -239,10 +263,13 @@ export const afterImportActions = {
     const db = getDatabaseFromConfig(config);
 
     const findCollectionId = async (collectionIdentifier: string) => {
-      const collections = await db.listCollections(dbId, [
-        Query.equal("name", collectionIdentifier),
-        Query.limit(1),
-      ]);
+      const collections = await tryAwaitWithRetry(
+        async () =>
+          await db.listCollections(dbId, [
+            Query.equal("name", collectionIdentifier),
+            Query.limit(1),
+          ])
+      );
       return collections.total > 0
         ? collections.collections[0].$id
         : collectionIdentifier;
@@ -279,10 +306,8 @@ export const afterImportActions = {
         if (cursor) {
           queries.push(Query.cursorAfter(cursor));
         }
-        const response = await db.listDocuments(
-          dbId,
-          otherCollectionId,
-          queries
+        const response = await tryAwaitWithRetry(
+          async () => await db.listDocuments(dbId, otherCollectionId, queries)
         );
         const documents = response.documents;
         if (documents.length === 0 || documents.length < docLimit) {
@@ -303,7 +328,15 @@ export const afterImportActions = {
         const updatePayload = targetFieldIsArray
           ? { [fieldName]: targetFieldValues }
           : { [fieldName]: targetFieldValues[0] };
-        await db.updateDocument(dbId, targetCollectionId, docId, updatePayload);
+        await tryAwaitWithRetry(
+          async () =>
+            await db.updateDocument(
+              dbId,
+              targetCollectionId,
+              docId,
+              updatePayload
+            )
+        );
 
         console.log(
           `Field ${fieldName} updated successfully in document ${docId} with values from field ${targetField}.`
@@ -331,40 +364,48 @@ export const afterImportActions = {
   ) => {
     try {
       const storage = getStorageFromConfig(config);
-      const bucket = await storage.listBuckets([
-        Query.equal("name", bucketName),
-      ]);
+      const bucket = await tryAwaitWithRetry(
+        async () => await storage.listBuckets([Query.equal("name", bucketName)])
+      );
       if (bucket.buckets.length > 0) {
         return bucket.buckets[0];
       } else if (bucketId) {
         try {
-          return await storage.getBucket(bucketId);
+          return await tryAwaitWithRetry(
+            async () => await storage.getBucket(bucketId)
+          );
         } catch (error) {
-          return await storage.createBucket(
-            bucketId,
-            bucketName,
-            permissions,
-            fileSecurity,
-            enabled,
-            maxFileSize,
-            allowedExtensions,
-            compression,
-            encryption,
-            antivirus
+          return await tryAwaitWithRetry(
+            async () =>
+              await storage.createBucket(
+                bucketId,
+                bucketName,
+                permissions,
+                fileSecurity,
+                enabled,
+                maxFileSize,
+                allowedExtensions,
+                compression,
+                encryption,
+                antivirus
+              )
           );
         }
       } else {
-        return await storage.createBucket(
-          bucketId || ID.unique(),
-          bucketName,
-          permissions,
-          fileSecurity,
-          enabled,
-          maxFileSize,
-          allowedExtensions,
-          compression,
-          encryption,
-          antivirus
+        return await tryAwaitWithRetry(
+          async () =>
+            await storage.createBucket(
+              bucketId || ID.unique(),
+              bucketName,
+              permissions,
+              fileSecurity,
+              enabled,
+              maxFileSize,
+              allowedExtensions,
+              compression,
+              encryption,
+              antivirus
+            )
         );
       }
     } catch (error) {
@@ -384,7 +425,9 @@ export const afterImportActions = {
     try {
       const db = getDatabaseFromConfig(config);
       const storage = getStorageFromConfig(config);
-      const collection = await db.getCollection(dbId, collId);
+      const collection = await tryAwaitWithRetry(
+        async () => await db.getCollection(dbId, collId)
+      );
       const attributes = collection.attributes as any[];
       const attribute = attributes.find((a) => a.key === fieldName);
       // console.log(
@@ -423,7 +466,9 @@ export const afterImportActions = {
         const tempFilePath = path.join(tempDir, fileName);
 
         // Download the file using fetch
-        const response = await fetch(filePath);
+        const response = await tryAwaitWithRetry(
+          async () => await fetch(filePath)
+        );
         if (!response.ok)
           console.error(
             `Failed to fetch ${filePath}: ${response.statusText} for document ${docId} with field ${fieldName}`

package/src/migrations/dataLoader.ts

@@ -362,7 +362,10 @@ export class DataLoader {
     console.log("---------------------------------");
     await this.setupMaps(dbId);
     const allUsers = await this.getAllUsers();
-    console.log(`Fetched ${allUsers.length} users`);
+    console.log(
+      `Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`
+    );
+    await new Promise((resolve) => setTimeout(resolve, 5000));
     // Iterate over the configured databases to find the matching one
     for (const db of this.config.databases) {
       if (db.$id !== dbId) {
@@ -940,9 +943,19 @@ export class DataLoader {
       );
 
       // Generate a new unique ID for the item or use existing ID
-      if (!existingId) {
+      if (!existingId && !userData.finalData?.userId) {
         // No existing user ID, generate a new unique ID
-        existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
+        existingId = this.getTrueUniqueId(
+          this.getCollectionKey(collection.name)
+        );
+        transformedItem = {
+          ...transformedItem,
+          userId: existingId,
+          docId: existingId,
+        };
+      } else if (!existingId && userData.finalData?.userId) {
+        // Existing user ID, use it as the new ID
+        existingId = userData.finalData.userId;
         transformedItem = {
           ...transformedItem,
           userId: existingId,
@@ -951,7 +964,7 @@ export class DataLoader {
       }
 
       // Create a context object for the item, including the new ID
-      let context = this.createContext(db, collection, item, existingId);
+      let context = this.createContext(db, collection, item, existingId!);
 
       // Merge the transformed data into the context
       context = { ...context, ...transformedItem, ...userData.finalData };
@@ -970,7 +983,9 @@ export class DataLoader {
           for (const data of currentData.data) {
             if (
               data.finalData.docId === oldId ||
-              data.finalData.userId === oldId
+              data.finalData.userId === oldId ||
+              data.context.docId === oldId ||
+              data.context.userId === oldId
             ) {
               transformedItem = this.mergeObjects(
                 data.finalData,
@@ -1022,41 +1037,51 @@ export class DataLoader {
         attributeMappings: mappingsWithActions,
       };
 
+      const updatedData = this.importMap.get(
+        this.getCollectionKey(collection.name)
+      )!;
+
       let foundData = false;
-      for (let i = 0; i < currentData.data.length; i++) {
+      for (let i = 0; i < updatedData.data.length; i++) {
         if (
-          currentData.data[i].finalData.docId === existingId ||
-          currentData.data[i].finalData.userId === existingId
+          updatedData.data[i].finalData.docId === existingId ||
+          updatedData.data[i].finalData.userId === existingId ||
+          updatedData.data[i].context.docId === existingId ||
+          updatedData.data[i].context.userId === existingId
         ) {
-          currentData.data[i].finalData = this.mergeObjects(
-            currentData.data[i].finalData,
+          updatedData.data[i].finalData = this.mergeObjects(
+            updatedData.data[i].finalData,
             transformedItem
           );
-          currentData.data[i].context = {
-            ...currentData.data[i].context,
-            ...context,
-          };
-          currentData.data[i].importDef = newImportDef;
+          updatedData.data[i].context = this.mergeObjects(
+            updatedData.data[i].context,
+            context
+          );
+          updatedData.data[i].importDef = this.mergeObjects(
+            updatedData.data[i].importDef,
+            newImportDef
+          );
           this.importMap.set(
             this.getCollectionKey(collection.name),
-            currentData
+            updatedData
           );
           this.oldIdToNewIdPerCollectionMap.set(
             this.getCollectionKey(collection.name),
             collectionOldIdToNewIdMap!
           );
+
           foundData = true;
         }
       }
       if (!foundData) {
         // Add new data to the associated collection
-        currentData.data.push({
+        updatedData.data.push({
           rawData: item,
           context: context,
           importDef: newImportDef,
           finalData: transformedItem,
         });
-        this.importMap.set(this.getCollectionKey(collection.name), currentData);
+        this.importMap.set(this.getCollectionKey(collection.name), updatedData);
         this.oldIdToNewIdPerCollectionMap.set(
           this.getCollectionKey(collection.name),
           collectionOldIdToNewIdMap!
@@ -1119,18 +1144,10 @@ export class DataLoader {
       // Create a context object for the item, including the new ID
       let context = this.createContext(db, collection, item, itemIdNew);
       // Transform the item data based on the attribute mappings
-      const transformedData = this.transformData(
+      let transformedData = this.transformData(
         item,
         importDef.attributeMappings
       );
-      if (collection.name.toLowerCase() === "councils") {
-        console.log("Transformed Council: ", transformedData);
-      }
-      if (isRegions) {
-        logger.info(
-          `Transformed region: ${JSON.stringify(transformedData, null, 2)}`
-        );
-      }
       // If a primary key field is defined, handle the ID mapping and check for duplicates
       if (importDef.primaryKeyField) {
         const oldId = item[importDef.primaryKeyField];

package/src/migrations/importController.ts

@@ -237,6 +237,7 @@ export class ImportController {
       const collectionData = dataLoader.importMap.get(
         dataLoader.getCollectionKey(collection.name)
       );
+      console.log(`Processing collection: ${collection.name}...`);
       if (!collectionData) {
         console.log("No collection data for ", collection.name);
         continue;

package/src/migrations/storage.ts

@@ -390,7 +390,9 @@ export const transferStorageLocalToLocal = async (
   let numberOfFiles = 0;
   if (fromFiles.files.length < 100) {
     for (const file of allFromFiles) {
-      const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+      const fileData = await tryAwaitWithRetry(
+        async () => await storage.getFileDownload(file.bucketId, file.$id)
+      );
       const fileToCreate = InputFile.fromBuffer(
         Buffer.from(fileData),
         file.name
@@ -410,10 +412,13 @@ export const transferStorageLocalToLocal = async (
   } else {
     lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
     while (lastFileId) {
-      const files = await storage.listFiles(fromBucketId, [
-        Query.limit(100),
-        Query.cursorAfter(lastFileId),
-      ]);
+      const files = await tryAwaitWithRetry(
+        async () =>
+          await storage.listFiles(fromBucketId, [
+            Query.limit(100),
+            Query.cursorAfter(lastFileId!),
+          ])
+      );
       allFromFiles.push(...files.files);
       if (files.files.length < 100) {
         lastFileId = undefined;
@@ -422,7 +427,9 @@ export const transferStorageLocalToLocal = async (
       }
     }
     for (const file of allFromFiles) {
-      const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+      const fileData = await tryAwaitWithRetry(
+        async () => await storage.getFileDownload(file.bucketId, file.$id)
+      );
       const fileToCreate = InputFile.fromBuffer(
         Buffer.from(fileData),
         file.name
@@ -467,10 +474,13 @@ export const transferStorageLocalToRemote = async (
   if (fromFiles.files.length === 100) {
     lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
     while (lastFileId) {
-      const files = await localStorage.listFiles(fromBucketId, [
-        Query.limit(100),
-        Query.cursorAfter(lastFileId),
-      ]);
+      const files = await tryAwaitWithRetry(
+        async () =>
+          await localStorage.listFiles(fromBucketId, [
+            Query.limit(100),
+            Query.cursorAfter(lastFileId!),
+          ])
+      );
       allFromFiles.push(...files.files);
       if (files.files.length < 100) {
         break;