appwrite-utils-cli 0.0.46 → 0.0.48

package/src/migrations/dataLoader.ts

@@ -393,7 +393,7 @@ export class DataLoader {
         }
       }
       console.log("Running update references");
-      // this.dealWithMergedUsers();
+      this.dealWithMergedUsers();
       this.updateOldReferencesForNew();
       console.log("Done running update references");
     }
@@ -1164,19 +1164,18 @@ export class DataLoader {
    * @param collection - The collection configuration.
    * @param importDef - The import definition containing the attribute mappings and other relevant info.
    */
-  prepareUpdateData(
+  async prepareUpdateData(
     db: ConfigDatabase,
     collection: CollectionCreate,
     importDef: ImportDef
   ) {
-    // Retrieve the current collection data and old-to-new ID map from the import map
     const currentData = this.importMap.get(
       this.getCollectionKey(collection.name)
     );
     const oldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(
       this.getCollectionKey(collection.name)
     );
-    // Log an error and return if no current data is found for the collection
+
     if (
       !(currentData?.data && currentData?.data.length > 0) &&
       !oldIdToNewIdMap
@@ -1186,7 +1185,7 @@ export class DataLoader {
       );
       return;
     }
-    // Load the raw data based on the import definition
+
     const rawData = this.loadData(importDef);
     const operationId = this.collectionImportOperations.get(
       this.getCollectionKey(collection.name)
@@ -1196,31 +1195,69 @@ export class DataLoader {
         `No import operation found for collection ${collection.name}`
       );
     }
+
     for (const item of rawData) {
-      // Transform the item data based on the attribute mappings
       let transformedData = this.transformData(
         item,
         importDef.attributeMappings
       );
       let newId: string | undefined;
       let oldId: string | undefined;
-      // Determine the new ID for the item based on the primary key field or update mapping
-      oldId = item[importDef.primaryKeyField];
-      if (oldId) {
-        newId = oldIdToNewIdMap?.get(`${oldId}`);
-        if (
-          !newId &&
-          this.getCollectionKey(this.config.usersCollectionName) ===
-            this.getCollectionKey(collection.name)
-        ) {
-          for (const [key, value] of this.mergedUserMap.entries()) {
-            if (value.includes(`${oldId}`)) {
-              newId = key;
-              break;
+      let itemDataToUpdate: CollectionImportData["data"][number] | undefined;
+
+      // Try to find itemDataToUpdate using updateMapping
+      if (importDef.updateMapping) {
+        oldId = item[importDef.updateMapping.originalIdField];
+        if (oldId) {
+          itemDataToUpdate = currentData?.data.find(
+            ({ context, rawData, finalData }) => {
+              const targetField =
+                importDef.updateMapping!.targetField ??
+                importDef.updateMapping!.originalIdField;
+              return (
+                `${context[targetField]}` === `${oldId}` ||
+                `${rawData[targetField]}` === `${oldId}` ||
+                `${finalData[targetField]}` === `${oldId}`
+              );
             }
+          );
+
+          if (itemDataToUpdate) {
+            newId =
+              itemDataToUpdate.finalData.docId ||
+              itemDataToUpdate.context.docId;
           }
         }
-      } else {
+      }
+
+      // If updateMapping is not defined or did not find the item, use primaryKeyField
+      if (!itemDataToUpdate && importDef.primaryKeyField) {
+        oldId = item[importDef.primaryKeyField];
+        if (oldId) {
+          newId = oldIdToNewIdMap?.get(`${oldId}`);
+          if (
+            !newId &&
+            this.getCollectionKey(this.config.usersCollectionName) ===
+              this.getCollectionKey(collection.name)
+          ) {
+            for (const [key, value] of this.mergedUserMap.entries()) {
+              if (value.includes(`${oldId}`)) {
+                newId = key;
+                break;
+              }
+            }
+          }
+        }
+
+        if (oldId && !itemDataToUpdate) {
+          itemDataToUpdate = currentData?.data.find(
+            (data) =>
+              `${data.context[importDef.primaryKeyField]}` === `${oldId}`
+          );
+        }
+      }
+
+      if (!oldId) {
         logger.error(
           `No old ID found (to update another document with) in prepareUpdateData for ${
             collection.name
@@ -1228,12 +1265,7 @@ export class DataLoader {
         );
         continue;
       }
-      const itemDataToUpdate = this.importMap
-        .get(this.getCollectionKey(collection.name))
-        ?.data.find(
-          (data) => `${data.context[importDef.primaryKeyField]}` === `${oldId}`
-        );
-      // Log an error and continue to the next item if no new ID is found
+
       if (!newId && !itemDataToUpdate) {
         logger.error(
           `No new id found for collection ${
@@ -1260,6 +1292,7 @@ export class DataLoader {
           continue;
         }
       }
+
       if (!itemDataToUpdate || !newId) {
         logger.error(
           `No data or ID (docId) found for collection ${
@@ -1272,49 +1305,51 @@ export class DataLoader {
         );
         continue;
       }
+
       transformedData = this.mergeObjects(
         itemDataToUpdate.finalData,
         transformedData
       );
+
       // Create a context object for the item, including the new ID and transformed data
       let context = this.createContext(db, collection, item, newId);
-      context = { ...context, ...transformedData };
+      context = this.mergeObjects(context, transformedData);
+
       // Validate the item before proceeding
-      const isValid = this.importDataActions.validateItem(
+      const isValid = await this.importDataActions.validateItem(
         item,
         importDef.attributeMappings,
         context
       );
-      // Log info and continue to the next item if it's invalid
+
       if (!isValid) {
         logger.info(
           `Skipping item: ${JSON.stringify(item, null, 2)} because it's invalid`
         );
         continue;
       }
+
       // Update the attribute mappings with any actions that need to be performed post-import
       const mappingsWithActions = this.getAttributeMappingsWithActions(
         importDef.attributeMappings,
         context,
         transformedData
       );
+
       // Update the import definition with the new attribute mappings
       const newImportDef = {
         ...importDef,
         attributeMappings: mappingsWithActions,
       };
-      // Add the item with its context and final data to the current collection data
+
       if (itemDataToUpdate) {
-        // Update the existing item's finalData and context in place
         itemDataToUpdate.finalData = this.mergeObjects(
           itemDataToUpdate.finalData,
           transformedData
         );
         itemDataToUpdate.context = context;
         itemDataToUpdate.importDef = newImportDef;
-        currentData!.data.push(itemDataToUpdate);
       } else {
-        // If no existing item matches, then add the new item
         currentData!.data.push({
           rawData: item,
           context: context,
@@ -1322,6 +1357,7 @@ export class DataLoader {
           finalData: transformedData,
         });
       }
+
       // Since we're modifying currentData in place, we ensure no duplicates are added
       this.importMap.set(this.getCollectionKey(collection.name), currentData!);
     }

package/src/migrations/databases.ts

@@ -1,5 +1,14 @@
-import { Databases, Query, type Models } from "node-appwrite";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { Client, Databases, Query, type Models } from "node-appwrite";
+import {
+  getAppwriteClient,
+  tryAwaitWithRetry,
+} from "../utils/helperFunctions.js";
+import {
+  transferDocumentsBetweenDbsLocalToLocal,
+  transferDocumentsBetweenDbsLocalToRemote,
+} from "./collections.js";
+import { createOrUpdateAttribute } from "./attributes.js";
+import { parseAttribute } from "appwrite-utils";
 
 export const fetchAllDatabases = async (
   database: Databases
@@ -26,3 +35,213 @@ export const fetchAllDatabases = async (
   }
   return allDatabases;
 };
+
+/**
+ * Transfers all collections and documents from one local database to another local database.
+ *
+ * @param {Databases} localDb - The local database instance.
+ * @param {string} fromDbId - The ID of the source database.
+ * @param {string} targetDbId - The ID of the target database.
+ * @return {Promise<void>} A promise that resolves when the transfer is complete.
+ */
+export const transferDatabaseLocalToLocal = async (
+  localDb: Databases,
+  fromDbId: string,
+  targetDbId: string
+) => {
+  let lastCollectionId: string | undefined;
+  let fromCollections = await tryAwaitWithRetry(
+    async () => await localDb.listCollections(fromDbId, [Query.limit(50)])
+  );
+  const allFromCollections = fromCollections.collections;
+  if (fromCollections.collections.length < 50) {
+    lastCollectionId = undefined;
+  } else {
+    lastCollectionId =
+      fromCollections.collections[fromCollections.collections.length - 1].$id;
+    while (lastCollectionId) {
+      const collections = await localDb.listCollections(fromDbId, [
+        Query.limit(50),
+        Query.cursorAfter(lastCollectionId),
+      ]);
+      allFromCollections.push(...collections.collections);
+      if (collections.collections.length < 50) {
+        break;
+      }
+      lastCollectionId =
+        collections.collections[collections.collections.length - 1].$id;
+    }
+  }
+  lastCollectionId = undefined;
+  let toCollections = await tryAwaitWithRetry(
+    async () => await localDb.listCollections(targetDbId, [Query.limit(50)])
+  );
+  const allToCollections = toCollections.collections;
+  if (toCollections.collections.length < 50) {
+  } else {
+    lastCollectionId =
+      toCollections.collections[toCollections.collections.length - 1].$id;
+    while (lastCollectionId) {
+      const collections = await localDb.listCollections(targetDbId, [
+        Query.limit(50),
+        Query.cursorAfter(lastCollectionId),
+      ]);
+      allToCollections.push(...collections.collections);
+      if (collections.collections.length < 50) {
+        lastCollectionId = undefined;
+      } else {
+        lastCollectionId =
+          collections.collections[collections.collections.length - 1].$id;
+      }
+    }
+  }
+  for (const collection of allFromCollections) {
+    const toCollection = allToCollections.find((c) => c.$id === collection.$id);
+    if (toCollection) {
+      await transferDocumentsBetweenDbsLocalToLocal(
+        localDb,
+        fromDbId,
+        targetDbId,
+        collection.$id,
+        toCollection.$id
+      );
+    } else {
+      console.log(
+        `Collection ${collection.name} not found in destination database, creating...`
+      );
+      const newCollection = await tryAwaitWithRetry(
+        async () =>
+          await localDb.createCollection(
+            targetDbId,
+            collection.$id,
+            collection.name,
+            collection.$permissions,
+            collection.documentSecurity,
+            collection.enabled
+          )
+      );
+      console.log(`Collection ${newCollection.name} created`);
+      for (const attribute of collection.attributes) {
+        await tryAwaitWithRetry(
+          async () =>
+            await createOrUpdateAttribute(
+              localDb,
+              targetDbId,
+              newCollection,
+              parseAttribute(attribute as any)
+            )
+        );
+      }
+      for (const index of collection.indexes) {
+        await tryAwaitWithRetry(
+          async () =>
+            await localDb.createIndex(
+              targetDbId,
+              newCollection.$id,
+              index.key,
+              index.type,
+              index.attributes,
+              index.orders
+            )
+        );
+      }
+      await transferDocumentsBetweenDbsLocalToLocal(
+        localDb,
+        fromDbId,
+        targetDbId,
+        collection.$id,
+        newCollection.$id
+      );
+    }
+  }
+};
+
+export const transferDatabaseLocalToRemote = async (
+  localDb: Databases,
+  endpoint: string,
+  projectId: string,
+  apiKey: string,
+  fromDbId: string,
+  toDbId: string
+) => {
+  const client = getAppwriteClient(endpoint, projectId, apiKey);
+  const remoteDb = new Databases(client);
+
+  let lastCollectionId: string | undefined;
+  let fromCollections = await tryAwaitWithRetry(
+    async () => await localDb.listCollections(fromDbId, [Query.limit(50)])
+  );
+  const allFromCollections = fromCollections.collections;
+  if (fromCollections.collections.length < 50) {
+  } else {
+    lastCollectionId =
+      fromCollections.collections[fromCollections.collections.length - 1].$id;
+    while (lastCollectionId) {
+      const collections = await tryAwaitWithRetry(
+        async () =>
+          await localDb.listCollections(fromDbId, [
+            Query.limit(50),
+            Query.cursorAfter(lastCollectionId!),
+          ])
+      );
+      allFromCollections.push(...collections.collections);
+      if (collections.collections.length < 50) {
+        break;
+      }
+      lastCollectionId =
+        collections.collections[collections.collections.length - 1].$id;
+    }
+  }
+
+  for (const collection of allFromCollections) {
+    const toCollection = await tryAwaitWithRetry(
+      async () =>
+        await remoteDb.createCollection(
+          toDbId,
+          collection.$id,
+          collection.name,
+          collection.$permissions,
+          collection.documentSecurity,
+          collection.enabled
+        )
+    );
+    console.log(`Collection ${toCollection.name} created`);
+
+    for (const attribute of collection.attributes) {
+      await tryAwaitWithRetry(
+        async () =>
+          await createOrUpdateAttribute(
+            remoteDb,
+            toDbId,
+            toCollection,
+            parseAttribute(attribute as any)
+          )
+      );
+    }
+
+    for (const index of collection.indexes) {
+      await tryAwaitWithRetry(
+        async () =>
+          await remoteDb.createIndex(
+            toDbId,
+            toCollection.$id,
+            index.key,
+            index.type,
+            index.attributes,
+            index.orders
+          )
+      );
+    }
+
+    await transferDocumentsBetweenDbsLocalToRemote(
+      localDb,
+      endpoint,
+      projectId,
+      apiKey,
+      fromDbId,
+      toDbId,
+      collection.$id,
+      toCollection.$id
+    );
+  }
+};

package/src/migrations/importController.ts

@@ -26,6 +26,9 @@ import {
   OperationSchema,
 } from "./backup.js";
 import { DataLoader, type CollectionImportData } from "./dataLoader.js";
+import { transferDocumentsBetweenDbsLocalToLocal } from "./collections.js";
+import { transferDatabaseLocalToLocal } from "./databases.js";
+import { transferStorageLocalToLocal } from "./storage.js";
 
 export class ImportController {
   private config: AppwriteConfig;
@@ -73,6 +76,7 @@ export class ImportController {
       )
       .map((db) => db.name);
     let dataLoader: DataLoader | undefined;
+    let databaseRan: ConfigDatabase | undefined;
     for (let db of this.config.databases) {
       if (
         db.name.toLowerCase().trim().replace(" ", "") === "migrations" ||
@@ -92,7 +96,8 @@ export class ImportController {
       console.log(`Starting import data for database: ${db.name}`);
       console.log(`---------------------------------`);
       // await this.importCollections(db);
-      if (!dataLoader) {
+      if (!databaseRan) {
+        databaseRan = db;
         dataLoader = new DataLoader(
           this.appwriteFolderPath,
           this.importDataActions,
@@ -101,18 +106,38 @@ export class ImportController {
           this.setupOptions.shouldWriteFile
         );
         await dataLoader.start(db.$id);
-      } else {
-        console.log(`Using data from previous import run`);
+        await this.importCollections(db, dataLoader);
+        await resolveAndUpdateRelationships(db.$id, this.database, this.config);
+        await this.executePostImportActions(db.$id, dataLoader);
+      } else if (databaseRan.$id !== db.$id) {
+        await this.updateOthersToFinalData(databaseRan, db);
       }
-      await this.importCollections(db, dataLoader);
-      await resolveAndUpdateRelationships(db.$id, this.database, this.config);
-      await this.executePostImportActions(db.$id, dataLoader);
       console.log(`---------------------------------`);
       console.log(`Finished import data for database: ${db.name}`);
       console.log(`---------------------------------`);
     }
   }
 
+  async updateOthersToFinalData(
+    updatedDb: ConfigDatabase,
+    targetDb: ConfigDatabase
+  ) {
+    await transferDatabaseLocalToLocal(
+      this.database,
+      updatedDb.$id,
+      targetDb.$id
+    );
+    await transferStorageLocalToLocal(
+      this.storage,
+      `${this.config.documentBucketId}_${updatedDb.name
+        .toLowerCase()
+        .replace(" ", "")}`,
+      `${this.config.documentBucketId}_${targetDb.name
+        .toLowerCase()
+        .replace(" ", "")}`
+    );
+  }
+
   async importCollections(db: ConfigDatabase, dataLoader: DataLoader) {
     if (!this.config.collections) {
       return;

package/src/migrations/storage.ts

@@ -10,7 +10,10 @@ import {
 import { type OperationCreate, type BackupCreate } from "./backup.js";
 import { splitIntoBatches } from "./migrationHelper.js";
 import type { AppwriteConfig } from "appwrite-utils";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import {
+  getAppwriteClient,
+  tryAwaitWithRetry,
+} from "../utils/helperFunctions.js";
 
 export const logOperation = async (
   db: Databases,
@@ -372,3 +375,123 @@ export const backupDatabase = async (
   console.log("Database Backup Complete");
   console.log("---------------------------------");
 };
+
+export const transferStorageLocalToLocal = async (
+  storage: Storage,
+  fromBucketId: string,
+  toBucketId: string
+) => {
+  console.log(`Transferring files from ${fromBucketId} to ${toBucketId}`);
+  let lastFileId: string | undefined;
+  let fromFiles = await tryAwaitWithRetry(
+    async () => await storage.listFiles(fromBucketId, [Query.limit(100)])
+  );
+  const allFromFiles = fromFiles.files;
+  let numberOfFiles = 0;
+  if (fromFiles.files.length < 100) {
+    for (const file of allFromFiles) {
+      const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+      const fileToCreate = InputFile.fromBuffer(
+        Buffer.from(fileData),
+        file.name
+      );
+      console.log(`Creating file: ${file.name}`);
+      tryAwaitWithRetry(
+        async () =>
+          await storage.createFile(
+            toBucketId,
+            file.$id,
+            fileToCreate,
+            file.$permissions
+          )
+      );
+      numberOfFiles++;
+    }
+  } else {
+    lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
+    while (lastFileId) {
+      const files = await storage.listFiles(fromBucketId, [
+        Query.limit(100),
+        Query.cursorAfter(lastFileId),
+      ]);
+      allFromFiles.push(...files.files);
+      if (files.files.length < 100) {
+        lastFileId = undefined;
+      } else {
+        lastFileId = files.files[files.files.length - 1].$id;
+      }
+    }
+    for (const file of allFromFiles) {
+      const fileData = await storage.getFileDownload(file.bucketId, file.$id);
+      const fileToCreate = InputFile.fromBuffer(
+        Buffer.from(fileData),
+        file.name
+      );
+      await tryAwaitWithRetry(
+        async () =>
+          await storage.createFile(
+            toBucketId,
+            file.$id,
+            fileToCreate,
+            file.$permissions
+          )
+      );
+      numberOfFiles++;
+    }
+  }
+
+  console.log(
+    `Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`
+  );
+};
+
+export const transferStorageLocalToRemote = async (
+  localStorage: Storage,
+  endpoint: string,
+  projectId: string,
+  apiKey: string,
+  fromBucketId: string,
+  toBucketId: string
+) => {
+  console.log(
+    `Transferring files from current storage ${fromBucketId} to ${endpoint} bucket ${toBucketId}`
+  );
+  const client = getAppwriteClient(endpoint, apiKey, projectId);
+  const remoteStorage = new Storage(client);
+  let numberOfFiles = 0;
+  let lastFileId: string | undefined;
+  let fromFiles = await tryAwaitWithRetry(
+    async () => await localStorage.listFiles(fromBucketId, [Query.limit(100)])
+  );
+  const allFromFiles = fromFiles.files;
+  if (fromFiles.files.length === 100) {
+    lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
+    while (lastFileId) {
+      const files = await localStorage.listFiles(fromBucketId, [
+        Query.limit(100),
+        Query.cursorAfter(lastFileId),
+      ]);
+      allFromFiles.push(...files.files);
+      if (files.files.length < 100) {
+        break;
+      }
+      lastFileId = files.files[files.files.length - 1].$id;
+    }
+  }
+
+  for (const file of allFromFiles) {
+    await tryAwaitWithRetry(
+      async () =>
+        await remoteStorage.createFile(
+          toBucketId,
+          file.$id,
+          file,
+          file.$permissions
+        )
+    );
+    numberOfFiles++;
+  }
+  console.log(
+    `Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`
+  );
+};