appwrite-utils-cli 0.0.62 → 0.0.63

package/dist/migrations/dataLoader.js

@@ -291,7 +291,8 @@ export class DataLoader {
         console.log("---------------------------------");
         await this.setupMaps(dbId);
         const allUsers = await this.getAllUsers();
-        console.log(`Fetched ${allUsers.length} users`);
+        console.log(`Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`);
+        await new Promise((resolve) => setTimeout(resolve, 5000));
         // Iterate over the configured databases to find the matching one
         for (const db of this.config.databases) {
             if (db.$id !== dbId) {
@@ -728,9 +729,18 @@ export class DataLoader {
             let { transformedItem, existingId, userData } = this.prepareUserData(item, importDef.attributeMappings, importDef.primaryKeyField, this.getTrueUniqueId(this.getCollectionKey("users")));
             logger.info(`In create user -- transformedItem: ${JSON.stringify(transformedItem, null, 2)}`);
             // Generate a new unique ID for the item or use existing ID
-            if (!existingId) {
+            if (!existingId && !userData.finalData?.userId) {
                 // No existing user ID, generate a new unique ID
-                existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
+                existingId = this.getTrueUniqueId(this.getCollectionKey(collection.name));
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
+            }
+            else if (!existingId && userData.finalData?.userId) {
+                // Existing user ID, use it as the new ID
+                existingId = userData.finalData.userId;
                 transformedItem = {
                     ...transformedItem,
                     userId: existingId,
@@ -751,7 +761,9 @@ export class DataLoader {
                     // Found a duplicate oldId, now decide how to merge or handle these duplicates
                     for (const data of currentData.data) {
                         if (data.finalData.docId === oldId ||
-                            data.finalData.userId === oldId) {
+                            data.finalData.userId === oldId ||
+                            data.context.docId === oldId ||
+                            data.context.userId === oldId) {
                             transformedItem = this.mergeObjects(data.finalData, transformedItem);
                         }
                     }
@@ -787,30 +799,30 @@ export class DataLoader {
                 ...importDef,
                 attributeMappings: mappingsWithActions,
             };
+            const updatedData = this.importMap.get(this.getCollectionKey(collection.name));
             let foundData = false;
-            for (let i = 0; i < currentData.data.length; i++) {
-                if (currentData.data[i].finalData.docId === existingId ||
-                    currentData.data[i].finalData.userId === existingId) {
-                    currentData.data[i].finalData = this.mergeObjects(currentData.data[i].finalData, transformedItem);
-                    currentData.data[i].context = {
-                        ...currentData.data[i].context,
-                        ...context,
-                    };
-                    currentData.data[i].importDef = newImportDef;
-                    this.importMap.set(this.getCollectionKey(collection.name), currentData);
+            for (let i = 0; i < updatedData.data.length; i++) {
+                if (updatedData.data[i].finalData.docId === existingId ||
+                    updatedData.data[i].finalData.userId === existingId ||
+                    updatedData.data[i].context.docId === existingId ||
+                    updatedData.data[i].context.userId === existingId) {
+                    updatedData.data[i].finalData = this.mergeObjects(updatedData.data[i].finalData, transformedItem);
+                    updatedData.data[i].context = this.mergeObjects(updatedData.data[i].context, context);
+                    updatedData.data[i].importDef = this.mergeObjects(updatedData.data[i].importDef, newImportDef);
+                    this.importMap.set(this.getCollectionKey(collection.name), updatedData);
                     this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
                     foundData = true;
                 }
             }
             if (!foundData) {
                 // Add new data to the associated collection
-                currentData.data.push({
+                updatedData.data.push({
                     rawData: item,
                     context: context,
                     importDef: newImportDef,
                     finalData: transformedItem,
                 });
-                this.importMap.set(this.getCollectionKey(collection.name), currentData);
+                this.importMap.set(this.getCollectionKey(collection.name), updatedData);
                 this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
             }
         }
@@ -854,13 +866,7 @@ export class DataLoader {
             // Create a context object for the item, including the new ID
             let context = this.createContext(db, collection, item, itemIdNew);
             // Transform the item data based on the attribute mappings
-            const transformedData = this.transformData(item, importDef.attributeMappings);
-            if (collection.name.toLowerCase() === "councils") {
-                console.log("Transformed Council: ", transformedData);
-            }
-            if (isRegions) {
-                logger.info(`Transformed region: ${JSON.stringify(transformedData, null, 2)}`);
-            }
+            let transformedData = this.transformData(item, importDef.attributeMappings);
             // If a primary key field is defined, handle the ID mapping and check for duplicates
             if (importDef.primaryKeyField) {
                 const oldId = item[importDef.primaryKeyField];

package/dist/migrations/importController.js

@@ -157,6 +157,7 @@ export class ImportController {
                 status: "in_progress",
             });
             const collectionData = dataLoader.importMap.get(dataLoader.getCollectionKey(collection.name));
+            console.log(`Processing collection: ${collection.name}...`);
             if (!collectionData) {
                 console.log("No collection data for ", collection.name);
                 continue;

package/dist/migrations/storage.js

@@ -253,7 +253,7 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
     let numberOfFiles = 0;
     if (fromFiles.files.length < 100) {
         for (const file of allFromFiles) {
-            const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+            const fileData = await tryAwaitWithRetry(async () => await storage.getFileDownload(file.bucketId, file.$id));
             const fileToCreate = InputFile.fromBuffer(Buffer.from(fileData), file.name);
             console.log(`Creating file: ${file.name}`);
             tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
@@ -263,10 +263,10 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
     else {
         lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
         while (lastFileId) {
-            const files = await storage.listFiles(fromBucketId, [
+            const files = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [
                 Query.limit(100),
                 Query.cursorAfter(lastFileId),
-            ]);
+            ]));
             allFromFiles.push(...files.files);
             if (files.files.length < 100) {
                 lastFileId = undefined;
@@ -276,7 +276,7 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
             }
         }
         for (const file of allFromFiles) {
-            const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+            const fileData = await tryAwaitWithRetry(async () => await storage.getFileDownload(file.bucketId, file.$id));
             const fileToCreate = InputFile.fromBuffer(Buffer.from(fileData), file.name);
             await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
             numberOfFiles++;
@@ -295,10 +295,10 @@ export const transferStorageLocalToRemote = async (localStorage, endpoint, proje
     if (fromFiles.files.length === 100) {
         lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
         while (lastFileId) {
-            const files = await localStorage.listFiles(fromBucketId, [
+            const files = await tryAwaitWithRetry(async () => await localStorage.listFiles(fromBucketId, [
                 Query.limit(100),
                 Query.cursorAfter(lastFileId),
-            ]);
+            ]));
             allFromFiles.push(...files.files);
             if (files.files.length < 100) {
                 break;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.62",
+  "version": "0.0.63",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/dataLoader.ts

@@ -362,7 +362,10 @@ export class DataLoader {
     console.log("---------------------------------");
     await this.setupMaps(dbId);
     const allUsers = await this.getAllUsers();
-    console.log(`Fetched ${allUsers.length} users`);
+    console.log(
+      `Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`
+    );
+    await new Promise((resolve) => setTimeout(resolve, 5000));
     // Iterate over the configured databases to find the matching one
     for (const db of this.config.databases) {
       if (db.$id !== dbId) {
@@ -940,9 +943,19 @@ export class DataLoader {
       );
 
       // Generate a new unique ID for the item or use existing ID
-      if (!existingId) {
+      if (!existingId && !userData.finalData?.userId) {
         // No existing user ID, generate a new unique ID
-        existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
+        existingId = this.getTrueUniqueId(
+          this.getCollectionKey(collection.name)
+        );
+        transformedItem = {
+          ...transformedItem,
+          userId: existingId,
+          docId: existingId,
+        };
+      } else if (!existingId && userData.finalData?.userId) {
+        // Existing user ID, use it as the new ID
+        existingId = userData.finalData.userId;
         transformedItem = {
           ...transformedItem,
           userId: existingId,
@@ -951,7 +964,7 @@ export class DataLoader {
       }
 
       // Create a context object for the item, including the new ID
-      let context = this.createContext(db, collection, item, existingId);
+      let context = this.createContext(db, collection, item, existingId!);
 
       // Merge the transformed data into the context
       context = { ...context, ...transformedItem, ...userData.finalData };
@@ -970,7 +983,9 @@ export class DataLoader {
           for (const data of currentData.data) {
             if (
               data.finalData.docId === oldId ||
-              data.finalData.userId === oldId
+              data.finalData.userId === oldId ||
+              data.context.docId === oldId ||
+              data.context.userId === oldId
             ) {
               transformedItem = this.mergeObjects(
                 data.finalData,
@@ -1022,41 +1037,51 @@ export class DataLoader {
         attributeMappings: mappingsWithActions,
       };
 
+      const updatedData = this.importMap.get(
+        this.getCollectionKey(collection.name)
+      )!;
+
       let foundData = false;
-      for (let i = 0; i < currentData.data.length; i++) {
+      for (let i = 0; i < updatedData.data.length; i++) {
         if (
-          currentData.data[i].finalData.docId === existingId ||
-          currentData.data[i].finalData.userId === existingId
+          updatedData.data[i].finalData.docId === existingId ||
+          updatedData.data[i].finalData.userId === existingId ||
+          updatedData.data[i].context.docId === existingId ||
+          updatedData.data[i].context.userId === existingId
         ) {
-          currentData.data[i].finalData = this.mergeObjects(
-            currentData.data[i].finalData,
+          updatedData.data[i].finalData = this.mergeObjects(
+            updatedData.data[i].finalData,
             transformedItem
           );
-          currentData.data[i].context = {
-            ...currentData.data[i].context,
-            ...context,
-          };
-          currentData.data[i].importDef = newImportDef;
+          updatedData.data[i].context = this.mergeObjects(
+            updatedData.data[i].context,
+            context
+          );
+          updatedData.data[i].importDef = this.mergeObjects(
+            updatedData.data[i].importDef,
+            newImportDef
+          );
           this.importMap.set(
             this.getCollectionKey(collection.name),
-            currentData
+            updatedData
           );
           this.oldIdToNewIdPerCollectionMap.set(
             this.getCollectionKey(collection.name),
             collectionOldIdToNewIdMap!
           );
+
           foundData = true;
         }
       }
       if (!foundData) {
         // Add new data to the associated collection
-        currentData.data.push({
+        updatedData.data.push({
           rawData: item,
           context: context,
           importDef: newImportDef,
           finalData: transformedItem,
         });
-        this.importMap.set(this.getCollectionKey(collection.name), currentData);
+        this.importMap.set(this.getCollectionKey(collection.name), updatedData);
         this.oldIdToNewIdPerCollectionMap.set(
           this.getCollectionKey(collection.name),
           collectionOldIdToNewIdMap!
@@ -1119,18 +1144,10 @@ export class DataLoader {
       // Create a context object for the item, including the new ID
       let context = this.createContext(db, collection, item, itemIdNew);
       // Transform the item data based on the attribute mappings
-      const transformedData = this.transformData(
+      let transformedData = this.transformData(
         item,
         importDef.attributeMappings
       );
-      if (collection.name.toLowerCase() === "councils") {
-        console.log("Transformed Council: ", transformedData);
-      }
-      if (isRegions) {
-        logger.info(
-          `Transformed region: ${JSON.stringify(transformedData, null, 2)}`
-        );
-      }
       // If a primary key field is defined, handle the ID mapping and check for duplicates
       if (importDef.primaryKeyField) {
         const oldId = item[importDef.primaryKeyField];

package/src/migrations/importController.ts

@@ -237,6 +237,7 @@ export class ImportController {
       const collectionData = dataLoader.importMap.get(
         dataLoader.getCollectionKey(collection.name)
       );
+      console.log(`Processing collection: ${collection.name}...`);
       if (!collectionData) {
         console.log("No collection data for ", collection.name);
         continue;

package/src/migrations/storage.ts

@@ -390,7 +390,9 @@ export const transferStorageLocalToLocal = async (
   let numberOfFiles = 0;
   if (fromFiles.files.length < 100) {
     for (const file of allFromFiles) {
-      const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+      const fileData = await tryAwaitWithRetry(
+        async () => await storage.getFileDownload(file.bucketId, file.$id)
+      );
       const fileToCreate = InputFile.fromBuffer(
         Buffer.from(fileData),
         file.name
@@ -410,10 +412,13 @@ export const transferStorageLocalToLocal = async (
   } else {
     lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
     while (lastFileId) {
-      const files = await storage.listFiles(fromBucketId, [
-        Query.limit(100),
-        Query.cursorAfter(lastFileId),
-      ]);
+      const files = await tryAwaitWithRetry(
+        async () =>
+          await storage.listFiles(fromBucketId, [
+            Query.limit(100),
+            Query.cursorAfter(lastFileId!),
+          ])
+      );
       allFromFiles.push(...files.files);
       if (files.files.length < 100) {
         lastFileId = undefined;
@@ -422,7 +427,9 @@ export const transferStorageLocalToLocal = async (
       }
     }
     for (const file of allFromFiles) {
-      const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+      const fileData = await tryAwaitWithRetry(
+        async () => await storage.getFileDownload(file.bucketId, file.$id)
+      );
       const fileToCreate = InputFile.fromBuffer(
         Buffer.from(fileData),
         file.name
@@ -467,10 +474,13 @@ export const transferStorageLocalToRemote = async (
   if (fromFiles.files.length === 100) {
     lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
     while (lastFileId) {
-      const files = await localStorage.listFiles(fromBucketId, [
-        Query.limit(100),
-        Query.cursorAfter(lastFileId),
-      ]);
+      const files = await tryAwaitWithRetry(
+        async () =>
+          await localStorage.listFiles(fromBucketId, [
+            Query.limit(100),
+            Query.cursorAfter(lastFileId!),
+          ])
+      );
       allFromFiles.push(...files.files);
       if (files.files.length < 100) {
         break;