appwrite-utils-cli 0.0.47 → 0.0.48

package/README.md

@@ -12,6 +12,7 @@
 - **Data Import**: Facilitate the import of data into your Appwrite databases with comprehensive command-line support.
 - **Backup Management**: Create backups of your Appwrite databases to ensure data integrity and safety.
 - **Flexible Database Management**: Includes commands to wipe databases, documents, or user data, providing flexibility in managing your database state during development or testing.
+- **Transfer Databases, Collections, Documents, Storage Buckets, and more**: Includes additional commands (new) to transfer your data from one place to another. I also optimized the import process using this.
 
 ## Installation
 
@@ -60,11 +61,56 @@ Replace `--args` with the appropriate options:
 - `--import`: Import data into your databases.
 - `--backup`: Perform a backup of your databases.
 - `--wipe-users`: Wipe all user data.
-- `--write-data`: Write converted imported data to file
-- `--sync`: Synchronize your project's config and generate schema for your database
-- `--endpoint`: Set a different endpoint for the migration target
-- `--project`: Set a different project ID for the migration target
-- `--key`: Set a different API key for the migration target
+- `--write-data`: Write converted imported data to file.
+- `--sync`: Synchronize your project's config and generate schema for your database.
+- `--endpoint <endpoint>`: Set the Appwrite endpoint.
+- `--project <project>`: Set the Appwrite project ID.
+- `--key <key>`: Set the Appwrite API key.
+- `--transfer`: Transfer documents between databases.
+- `--transfer-users`: Transfer users between local and remote.
+- `--transferendpoint <transferEndpoint>`: Set the transfer endpoint for remote transfers.
+- `--transferproject <transferProject>`: Set the transfer project ID for remote transfers.
+- `--transferkey <transferKey>`: Set the transfer key for remote transfers.
+- `--fromdb <fromDbId>`: Set the source database ID.
+- `--targetdb <targetDbId>`: Set the destination database ID.
+- `--fromcoll <collectionId>`: Set the source collection ID for transfer, only used for transfer.
+- `--targetcoll <collectionId>`: Set the collection ID to import data into.
+- `--frombucket <bucketId>`: Set the source bucket ID.
+- `--targetbucket <bucketId>`: Set the destination bucket ID.
+
+## Examples
+
+### Transfer Databases
+
+Transfer databases within the same project or from a local to a remote project. If `--fromcoll` and `--targetcoll` are omitted, it will transfer the entire databases. During the database transfer, it will create any missing collections, attributes, and indices.
+
+```bash
+npx appwrite-utils-cli appwrite-migrate --transfer --fromdb fromDbId --targetdb toDbId --transferendpoint https://appwrite.otherserver.com --transferproject yourProjectId --transferkey yourApiKey
+```
+
+### Transfer Specific Collections
+
+Transfer specific collections from one place to another, with all of their data.
+
+```bash
+npx appwrite-utils-cli appwrite-migrate --transfer --fromdb fromDbId --targetdb toDbId --fromcoll sourceCollectionId --targetcoll targetCollectionId --transferendpoint https://appwrite.otherserver.com --transferproject yourProjectId --transferkey yourApiKey
+```
+
+### Transfer Buckets
+
+Transfer files between buckets.
+
+```bash
+npx appwrite-utils-cli appwrite-migrate --transfer --frombucket sourceBucketId --targetbucket targetBucketId --transferendpoint https://appwrite.otherserver.com --transferproject yourProjectId --transferkey yourApiKey
+```
+
+### Transfer Users
+
+Transfer users between local and remote.
+
+```bash
+npx appwrite-utils-cli appwrite-migrate --transfer-users --transferendpoint https://appwrite.otherserver.com --transferproject yourProjectId --transferkey yourApiKey
+```
 
 ## If you run out of RAM
 
@@ -86,7 +132,8 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 ### Changelog
 
-- 0.0.48: Minor bugfixes in many releases, too small to take note of
+- 0.0.48: Added `--transfer`, `--fromdb <targetDatabaseId>`, `--targetdb <targetDatabaseId>`, `--transferendpoint <transferEndpoint>`, `--transferproject <transferProjectId>`, `--transferkey <transferApiKey>`. Additionally, I've added `--fromcoll <collectionId>` and `--targetcoll <collectionId>`. These allow you to do a few things. First, you can now transfer databases in the same project, and from local to a remote project. Second, you can now specify specific collections to transfer from one place to another, with all of their data. If `--fromcoll` and `--targetcoll` are ommitted, it will transfer the databases. During the database transfer, it will create any missing collections, attributes, and indices.
+- 0.0.47: Minor bugfixes in many releases, too small to take note of
 - 0.0.38: Lots of optimizations done to the code, added `tryAwaitWithRetry` for `fetch failed` and others like it errors (looking at you `server error`) -- this should prevent things from going sideways.
 - 0.0.37: Added `documentSecurity`, `enabled`, and `$id` to the `init` collection
 - 0.0.36: Made it update collections by default, sometimes you gotta do what you gotta do

package/dist/main.js

@@ -8,6 +8,17 @@ program
     .option("--endpoint <endpoint>", "Set the Appwrite endpoint", undefined)
     .option("--project <project>", "Set the Appwrite project ID", undefined)
     .option("--key <key>", "Set the Appwrite API key", undefined)
+    .option("--transfer", "Transfer documents between databases", false)
+    .option("--transfer-users", "Transfer users between local and remote", false)
+    .option("--transferendpoint <transferEndpoint>", "Set the transfer endpoint for remote transfers", undefined)
+    .option("--transferproject <transferProject>", "Set the transfer project ID for remote transfers", undefined)
+    .option("--transferkey <transferKey>", "Set the transfer key for remote transfers", undefined)
+    .option("--fromdb <fromDbId>", "Set the source database ID", undefined)
+    .option("--targetdb <targetDbId>", "Set the destination database ID", undefined)
+    .option("--fromcoll <collectionId>", "Set the source collection ID for transfer, only used for transfer", undefined)
+    .option("--targetcoll <collectionId>", "Set the collection ID to import data into", undefined)
+    .option("--frombucket <bucketId>", "Set the source bucket ID", undefined)
+    .option("--targetbucket <bucketId>", "Set the destination bucket ID", undefined)
     .option("--backup", "Perform a backup before executing the command", false)
     .option("--dev", "Run in development environment", false)
     .option("--prod", "Run in production environment", false)
@@ -24,6 +35,7 @@ program.on("--help", () => {
     console.log("");
     console.log("Examples:");
     console.log("  $ npx appwrite-utils-cli appwrite-migrate --sync --endpoint https://appwrite.example.com --project 123456 --key 7890");
+    console.log("  $ npx appwrite-utils-cli appwrite-migrate --transfer --fromdb fromDbId --targetdb toDbId --transferendpoint https://appwrite.otherserver.com --transferproject yourProjectId --transferkey yourApiKey");
     console.log("  $ npx appwrite-utils-cli appwrite-migrate --sync --dev --backup");
     console.log("  $ npx appwrite-utils-cli appwrite-migrate --wipe --wipe-docs --wipe-users --dev");
     console.log("  $ npx appwrite-utils-cli appwrite-migrate --generate --import --write-data --dev");
@@ -55,6 +67,17 @@ program.action(async (options) => {
             endpoint: options.endpoint,
             project: options.project,
             key: options.key,
+            transfer: options.transfer,
+            transferEndpoint: options.transferEndpoint,
+            transferProject: options.transferProject,
+            transferKey: options.transferKey,
+            fromDbId: options.fromdb,
+            targetDbId: options.targetdb,
+            fromCollection: options.fromcoll,
+            collection: options.targetcoll, // Add this line
+            transferUsers: options.transferUsers,
+            fromBucket: options.frombucket,
+            targetBucket: options.targetbucket,
         };
         console.log("Running operation...", setupOptions);
         await controller.run(setupOptions);

package/dist/migrations/collections.d.ts

@@ -14,3 +14,9 @@ export declare const createOrUpdateCollections: (database: Databases, databaseId
 }[]) => Promise<void>;
 export declare const generateMockData: (database: Databases, databaseId: string, configCollections: any[]) => Promise<void>;
 export declare const fetchAllCollections: (dbId: string, database: Databases) => Promise<Models.Collection[]>;
+/**
+ * Transfers all documents from one collection to another in a different database
+ * within the same Appwrite Project
+ */
+export declare const transferDocumentsBetweenDbsLocalToLocal: (db: Databases, fromDbId: string, toDbId: string, fromCollId: string, toCollId: string) => Promise<void>;
+export declare const transferDocumentsBetweenDbsLocalToRemote: (localDb: Databases, endpoint: string, projectId: string, apiKey: string, fromDbId: string, toDbId: string, fromCollId: string, toCollId: string) => Promise<void>;

package/dist/migrations/collections.js

@@ -1,4 +1,4 @@
-import { Databases, ID, Permission, Query } from "node-appwrite";
+import { Client, Databases, ID, Permission, Query, } from "node-appwrite";
 import { nameToIdMapping, processQueue } from "./queue.js";
 import { createUpdateCollectionAttributes } from "./attributes.js";
 import { createOrUpdateIndexes } from "./indexes.js";
@@ -96,7 +96,7 @@ export const wipeDatabase = async (database, databaseId) => {
             collectionId: collectionId,
             collectionName: name,
         });
-        await database.deleteCollection(databaseId, collectionId);
+        tryAwaitWithRetry(async () => await database.deleteCollection(databaseId, collectionId)); // Try to delete the collection and ignore errors if it doesn't exist or if it's already being deleted
     }
     return collectionsDeleted;
 };
@@ -214,3 +214,134 @@ export const fetchAllCollections = async (dbId, database) => {
     console.log(`Fetched a total of ${collections.length} collections.`);
     return collections;
 };
+/**
+ * Transfers all documents from one collection to another in a different database
+ * within the same Appwrite Project
+ */
+export const transferDocumentsBetweenDbsLocalToLocal = async (db, fromDbId, toDbId, fromCollId, toCollId) => {
+    let fromCollDocs = await tryAwaitWithRetry(async () => db.listDocuments(fromDbId, fromCollId, [Query.limit(50)]));
+    let totalDocumentsTransferred = 0;
+    if (fromCollDocs.documents.length === 0) {
+        console.log(`No documents found in collection ${fromCollId}`);
+        return;
+    }
+    else if (fromCollDocs.documents.length < 50) {
+        const batchedPromises = fromCollDocs.documents.map((doc) => {
+            const toCreateObject = {
+                ...doc,
+            };
+            delete toCreateObject.$databaseId;
+            delete toCreateObject.$collectionId;
+            delete toCreateObject.$createdAt;
+            delete toCreateObject.$updatedAt;
+            delete toCreateObject.$id;
+            delete toCreateObject.$permissions;
+            return tryAwaitWithRetry(async () => await db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+        });
+        await Promise.all(batchedPromises);
+        totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+    else {
+        const batchedPromises = fromCollDocs.documents.map((doc) => {
+            const toCreateObject = {
+                ...doc,
+            };
+            delete toCreateObject.$databaseId;
+            delete toCreateObject.$collectionId;
+            delete toCreateObject.$createdAt;
+            delete toCreateObject.$updatedAt;
+            delete toCreateObject.$id;
+            delete toCreateObject.$permissions;
+            return tryAwaitWithRetry(async () => db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+        });
+        await Promise.all(batchedPromises);
+        totalDocumentsTransferred += fromCollDocs.documents.length;
+        while (fromCollDocs.documents.length === 50) {
+            fromCollDocs = await db.listDocuments(fromDbId, fromCollId, [
+                Query.limit(50),
+                Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
+            ]);
+            const batchedPromises = fromCollDocs.documents.map((doc) => {
+                const toCreateObject = {
+                    ...doc,
+                };
+                delete toCreateObject.$databaseId;
+                delete toCreateObject.$collectionId;
+                delete toCreateObject.$createdAt;
+                delete toCreateObject.$updatedAt;
+                delete toCreateObject.$id;
+                delete toCreateObject.$permissions;
+                return tryAwaitWithRetry(async () => await db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+            });
+            await Promise.all(batchedPromises);
+            totalDocumentsTransferred += fromCollDocs.documents.length;
+        }
+    }
+    console.log(`Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`);
+};
+export const transferDocumentsBetweenDbsLocalToRemote = async (localDb, endpoint, projectId, apiKey, fromDbId, toDbId, fromCollId, toCollId) => {
+    const client = new Client()
+        .setEndpoint(endpoint)
+        .setProject(projectId)
+        .setKey(apiKey);
+    let totalDocumentsTransferred = 0;
+    const remoteDb = new Databases(client);
+    let fromCollDocs = await tryAwaitWithRetry(async () => localDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)]));
+    if (fromCollDocs.documents.length === 0) {
+        console.log(`No documents found in collection ${fromCollId}`);
+        return;
+    }
+    else if (fromCollDocs.documents.length < 50) {
+        const batchedPromises = fromCollDocs.documents.map((doc) => {
+            const toCreateObject = {
+                ...doc,
+            };
+            delete toCreateObject.$databaseId;
+            delete toCreateObject.$collectionId;
+            delete toCreateObject.$createdAt;
+            delete toCreateObject.$updatedAt;
+            delete toCreateObject.$id;
+            delete toCreateObject.$permissions;
+            return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+        });
+        await Promise.all(batchedPromises);
+        totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+    else {
+        const batchedPromises = fromCollDocs.documents.map((doc) => {
+            const toCreateObject = {
+                ...doc,
+            };
+            delete toCreateObject.$databaseId;
+            delete toCreateObject.$collectionId;
+            delete toCreateObject.$createdAt;
+            delete toCreateObject.$updatedAt;
+            delete toCreateObject.$id;
+            delete toCreateObject.$permissions;
+            return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+        });
+        await Promise.all(batchedPromises);
+        totalDocumentsTransferred += fromCollDocs.documents.length;
+        while (fromCollDocs.documents.length === 50) {
+            fromCollDocs = await tryAwaitWithRetry(async () => localDb.listDocuments(fromDbId, fromCollId, [
+                Query.limit(50),
+                Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
+            ]));
+            const batchedPromises = fromCollDocs.documents.map((doc) => {
+                const toCreateObject = {
+                    ...doc,
+                };
+                delete toCreateObject.$databaseId;
+                delete toCreateObject.$collectionId;
+                delete toCreateObject.$createdAt;
+                delete toCreateObject.$updatedAt;
+                delete toCreateObject.$id;
+                delete toCreateObject.$permissions;
+                return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+            });
+            await Promise.all(batchedPromises);
+            totalDocumentsTransferred += fromCollDocs.documents.length;
+        }
+    }
+    console.log(`Total documents transferred from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}: ${totalDocumentsTransferred}`);
+};

package/dist/migrations/dataLoader.d.ts

@@ -278,6 +278,16 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 originalIdField: z.ZodString;
                 targetField: z.ZodOptional<z.ZodString>;
             }, "strip", z.ZodTypeAny, {
+                /**
+                 * Prepares the data for updating documents within a collection.
+                 * This method loads the raw data based on the import definition, transforms it according to the attribute mappings,
+                 * finds the new ID for each item based on the primary key or update mapping, and then validates the transformed data.
+                 * If the data is valid, it updates the import definition with any post-import actions and adds the item to the current collection data.
+                 *
+                 * @param db - The database configuration.
+                 * @param collection - The collection configuration.
+                 * @param importDef - The import definition containing the attribute mappings and other relevant info.
+                 */
                 originalIdField: string;
                 targetField?: string | undefined;
             }, {
@@ -1767,7 +1777,7 @@ export declare class DataLoader {
      * @param collection - The collection configuration.
      * @param importDef - The import definition containing the attribute mappings and other relevant info.
      */
-    prepareUpdateData(db: ConfigDatabase, collection: CollectionCreate, importDef: ImportDef): void;
+    prepareUpdateData(db: ConfigDatabase, collection: CollectionCreate, importDef: ImportDef): Promise<void>;
     private updateReferencesBasedOnAttributeMappings;
     private getMergedId;
     /**

package/dist/migrations/dataLoader.js

@@ -314,7 +314,7 @@ export class DataLoader {
                 }
             }
             console.log("Running update references");
-            // this.dealWithMergedUsers();
+            this.dealWithMergedUsers();
             this.updateOldReferencesForNew();
             console.log("Done running update references");
         }
@@ -858,50 +858,66 @@ export class DataLoader {
      * @param collection - The collection configuration.
      * @param importDef - The import definition containing the attribute mappings and other relevant info.
      */
-    prepareUpdateData(db, collection, importDef) {
-        // Retrieve the current collection data and old-to-new ID map from the import map
+    async prepareUpdateData(db, collection, importDef) {
         const currentData = this.importMap.get(this.getCollectionKey(collection.name));
         const oldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(this.getCollectionKey(collection.name));
-        // Log an error and return if no current data is found for the collection
         if (!(currentData?.data && currentData?.data.length > 0) &&
             !oldIdToNewIdMap) {
             logger.error(`No data found for collection ${collection.name} for updateDef but it says it's supposed to have one...`);
             return;
         }
-        // Load the raw data based on the import definition
         const rawData = this.loadData(importDef);
         const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
         if (!operationId) {
             throw new Error(`No import operation found for collection ${collection.name}`);
         }
         for (const item of rawData) {
-            // Transform the item data based on the attribute mappings
             let transformedData = this.transformData(item, importDef.attributeMappings);
             let newId;
             let oldId;
-            // Determine the new ID for the item based on the primary key field or update mapping
-            oldId = item[importDef.primaryKeyField];
-            if (oldId) {
-                newId = oldIdToNewIdMap?.get(`${oldId}`);
-                if (!newId &&
-                    this.getCollectionKey(this.config.usersCollectionName) ===
-                        this.getCollectionKey(collection.name)) {
-                    for (const [key, value] of this.mergedUserMap.entries()) {
-                        if (value.includes(`${oldId}`)) {
-                            newId = key;
-                            break;
+            let itemDataToUpdate;
+            // Try to find itemDataToUpdate using updateMapping
+            if (importDef.updateMapping) {
+                oldId = item[importDef.updateMapping.originalIdField];
+                if (oldId) {
+                    itemDataToUpdate = currentData?.data.find(({ context, rawData, finalData }) => {
+                        const targetField = importDef.updateMapping.targetField ??
+                            importDef.updateMapping.originalIdField;
+                        return (`${context[targetField]}` === `${oldId}` ||
+                            `${rawData[targetField]}` === `${oldId}` ||
+                            `${finalData[targetField]}` === `${oldId}`);
+                    });
+                    if (itemDataToUpdate) {
+                        newId =
+                            itemDataToUpdate.finalData.docId ||
+                                itemDataToUpdate.context.docId;
+                    }
+                }
+            }
+            // If updateMapping is not defined or did not find the item, use primaryKeyField
+            if (!itemDataToUpdate && importDef.primaryKeyField) {
+                oldId = item[importDef.primaryKeyField];
+                if (oldId) {
+                    newId = oldIdToNewIdMap?.get(`${oldId}`);
+                    if (!newId &&
+                        this.getCollectionKey(this.config.usersCollectionName) ===
+                            this.getCollectionKey(collection.name)) {
+                        for (const [key, value] of this.mergedUserMap.entries()) {
+                            if (value.includes(`${oldId}`)) {
+                                newId = key;
+                                break;
+                            }
                         }
                     }
                 }
+                if (oldId && !itemDataToUpdate) {
+                    itemDataToUpdate = currentData?.data.find((data) => `${data.context[importDef.primaryKeyField]}` === `${oldId}`);
+                }
             }
-            else {
+            if (!oldId) {
                 logger.error(`No old ID found (to update another document with) in prepareUpdateData for ${collection.name}, ${JSON.stringify(item, null, 2)}`);
                 continue;
             }
-            const itemDataToUpdate = this.importMap
-                .get(this.getCollectionKey(collection.name))
-                ?.data.find((data) => `${data.context[importDef.primaryKeyField]}` === `${oldId}`);
-            // Log an error and continue to the next item if no new ID is found
             if (!newId && !itemDataToUpdate) {
                 logger.error(`No new id found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
                 continue;
@@ -920,10 +936,9 @@ export class DataLoader {
             transformedData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
             // Create a context object for the item, including the new ID and transformed data
             let context = this.createContext(db, collection, item, newId);
-            context = { ...context, ...transformedData };
+            context = this.mergeObjects(context, transformedData);
             // Validate the item before proceeding
-            const isValid = this.importDataActions.validateItem(item, importDef.attributeMappings, context);
-            // Log info and continue to the next item if it's invalid
+            const isValid = await this.importDataActions.validateItem(item, importDef.attributeMappings, context);
             if (!isValid) {
                 logger.info(`Skipping item: ${JSON.stringify(item, null, 2)} because it's invalid`);
                 continue;
@@ -935,16 +950,12 @@ export class DataLoader {
                 ...importDef,
                 attributeMappings: mappingsWithActions,
             };
-            // Add the item with its context and final data to the current collection data
             if (itemDataToUpdate) {
-                // Update the existing item's finalData and context in place
                 itemDataToUpdate.finalData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
                 itemDataToUpdate.context = context;
                 itemDataToUpdate.importDef = newImportDef;
-                currentData.data.push(itemDataToUpdate);
             }
             else {
-                // If no existing item matches, then add the new item
                 currentData.data.push({
                     rawData: item,
                     context: context,

package/dist/migrations/databases.d.ts

@@ -1,2 +1,12 @@
 import { Databases, type Models } from "node-appwrite";
 export declare const fetchAllDatabases: (database: Databases) => Promise<Models.Database[]>;
+/**
+ * Transfers all collections and documents from one local database to another local database.
+ *
+ * @param {Databases} localDb - The local database instance.
+ * @param {string} fromDbId - The ID of the source database.
+ * @param {string} targetDbId - The ID of the target database.
+ * @return {Promise<void>} A promise that resolves when the transfer is complete.
+ */
+export declare const transferDatabaseLocalToLocal: (localDb: Databases, fromDbId: string, targetDbId: string) => Promise<void>;
+export declare const transferDatabaseLocalToRemote: (localDb: Databases, endpoint: string, projectId: string, apiKey: string, fromDbId: string, toDbId: string) => Promise<void>;

package/dist/migrations/databases.js

@@ -1,5 +1,8 @@
-import { Databases, Query } from "node-appwrite";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { Client, Databases, Query } from "node-appwrite";
+import { getAppwriteClient, tryAwaitWithRetry, } from "../utils/helperFunctions.js";
+import { transferDocumentsBetweenDbsLocalToLocal, transferDocumentsBetweenDbsLocalToRemote, } from "./collections.js";
+import { createOrUpdateAttribute } from "./attributes.js";
+import { parseAttribute } from "appwrite-utils";
 export const fetchAllDatabases = async (database) => {
     const databases = await tryAwaitWithRetry(async () => await database.list([Query.limit(25)]));
     const allDatabases = databases.databases;
@@ -22,3 +25,112 @@ export const fetchAllDatabases = async (database) => {
     }
     return allDatabases;
 };
+/**
+ * Transfers all collections and documents from one local database to another local database.
+ *
+ * @param {Databases} localDb - The local database instance.
+ * @param {string} fromDbId - The ID of the source database.
+ * @param {string} targetDbId - The ID of the target database.
+ * @return {Promise<void>} A promise that resolves when the transfer is complete.
+ */
+export const transferDatabaseLocalToLocal = async (localDb, fromDbId, targetDbId) => {
+    let lastCollectionId;
+    let fromCollections = await tryAwaitWithRetry(async () => await localDb.listCollections(fromDbId, [Query.limit(50)]));
+    const allFromCollections = fromCollections.collections;
+    if (fromCollections.collections.length < 50) {
+        lastCollectionId = undefined;
+    }
+    else {
+        lastCollectionId =
+            fromCollections.collections[fromCollections.collections.length - 1].$id;
+        while (lastCollectionId) {
+            const collections = await localDb.listCollections(fromDbId, [
+                Query.limit(50),
+                Query.cursorAfter(lastCollectionId),
+            ]);
+            allFromCollections.push(...collections.collections);
+            if (collections.collections.length < 50) {
+                break;
+            }
+            lastCollectionId =
+                collections.collections[collections.collections.length - 1].$id;
+        }
+    }
+    lastCollectionId = undefined;
+    let toCollections = await tryAwaitWithRetry(async () => await localDb.listCollections(targetDbId, [Query.limit(50)]));
+    const allToCollections = toCollections.collections;
+    if (toCollections.collections.length < 50) {
+    }
+    else {
+        lastCollectionId =
+            toCollections.collections[toCollections.collections.length - 1].$id;
+        while (lastCollectionId) {
+            const collections = await localDb.listCollections(targetDbId, [
+                Query.limit(50),
+                Query.cursorAfter(lastCollectionId),
+            ]);
+            allToCollections.push(...collections.collections);
+            if (collections.collections.length < 50) {
+                lastCollectionId = undefined;
+            }
+            else {
+                lastCollectionId =
+                    collections.collections[collections.collections.length - 1].$id;
+            }
+        }
+    }
+    for (const collection of allFromCollections) {
+        const toCollection = allToCollections.find((c) => c.$id === collection.$id);
+        if (toCollection) {
+            await transferDocumentsBetweenDbsLocalToLocal(localDb, fromDbId, targetDbId, collection.$id, toCollection.$id);
+        }
+        else {
+            console.log(`Collection ${collection.name} not found in destination database, creating...`);
+            const newCollection = await tryAwaitWithRetry(async () => await localDb.createCollection(targetDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
+            console.log(`Collection ${newCollection.name} created`);
+            for (const attribute of collection.attributes) {
+                await tryAwaitWithRetry(async () => await createOrUpdateAttribute(localDb, targetDbId, newCollection, parseAttribute(attribute)));
+            }
+            for (const index of collection.indexes) {
+                await tryAwaitWithRetry(async () => await localDb.createIndex(targetDbId, newCollection.$id, index.key, index.type, index.attributes, index.orders));
+            }
+            await transferDocumentsBetweenDbsLocalToLocal(localDb, fromDbId, targetDbId, collection.$id, newCollection.$id);
+        }
+    }
+};
+export const transferDatabaseLocalToRemote = async (localDb, endpoint, projectId, apiKey, fromDbId, toDbId) => {
+    const client = getAppwriteClient(endpoint, projectId, apiKey);
+    const remoteDb = new Databases(client);
+    let lastCollectionId;
+    let fromCollections = await tryAwaitWithRetry(async () => await localDb.listCollections(fromDbId, [Query.limit(50)]));
+    const allFromCollections = fromCollections.collections;
+    if (fromCollections.collections.length < 50) {
+    }
+    else {
+        lastCollectionId =
+            fromCollections.collections[fromCollections.collections.length - 1].$id;
+        while (lastCollectionId) {
+            const collections = await tryAwaitWithRetry(async () => await localDb.listCollections(fromDbId, [
+                Query.limit(50),
+                Query.cursorAfter(lastCollectionId),
+            ]));
+            allFromCollections.push(...collections.collections);
+            if (collections.collections.length < 50) {
+                break;
+            }
+            lastCollectionId =
+                collections.collections[collections.collections.length - 1].$id;
+        }
+    }
+    for (const collection of allFromCollections) {
+        const toCollection = await tryAwaitWithRetry(async () => await remoteDb.createCollection(toDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
+        console.log(`Collection ${toCollection.name} created`);
+        for (const attribute of collection.attributes) {
+            await tryAwaitWithRetry(async () => await createOrUpdateAttribute(remoteDb, toDbId, toCollection, parseAttribute(attribute)));
+        }
+        for (const index of collection.indexes) {
+            await tryAwaitWithRetry(async () => await remoteDb.createIndex(toDbId, toCollection.$id, index.key, index.type, index.attributes, index.orders));
+        }
+        await transferDocumentsBetweenDbsLocalToRemote(localDb, endpoint, projectId, apiKey, fromDbId, toDbId, collection.$id, toCollection.$id);
+    }
+};

package/dist/migrations/importController.d.ts

@@ -16,6 +16,7 @@ export declare class ImportController {
     private postImportActionsQueue;
     constructor(config: AppwriteConfig, database: Databases, storage: Storage, appwriteFolderPath: string, importDataActions: ImportDataActions, setupOptions: SetupOptions);
     run(): Promise<void>;
+    updateOthersToFinalData(updatedDb: ConfigDatabase, targetDb: ConfigDatabase): Promise<void>;
     importCollections(db: ConfigDatabase, dataLoader: DataLoader): Promise<void>;
     executePostImportActions(dbId: string, dataLoader: DataLoader): Promise<void>;
 }

package/dist/migrations/importController.js

@@ -7,6 +7,9 @@ import { logger } from "./logging.js";
 import { updateOperation } from "./migrationHelper.js";
 import { BatchSchema, OperationCreateSchema, OperationSchema, } from "./backup.js";
 import { DataLoader } from "./dataLoader.js";
+import { transferDocumentsBetweenDbsLocalToLocal } from "./collections.js";
+import { transferDatabaseLocalToLocal } from "./databases.js";
+import { transferStorageLocalToLocal } from "./storage.js";
 export class ImportController {
     config;
     database;
@@ -37,6 +40,7 @@ export class ImportController {
                 this.setupOptions.runDev))
             .map((db) => db.name);
         let dataLoader;
+        let databaseRan;
         for (let db of this.config.databases) {
             if (db.name.toLowerCase().trim().replace(" ", "") === "migrations" ||
                 !databasesToRun.includes(db.name)) {
@@ -54,16 +58,30 @@ export class ImportController {
             console.log(`Starting import data for database: ${db.name}`);
             console.log(`---------------------------------`);
             // await this.importCollections(db);
-            dataLoader = new DataLoader(this.appwriteFolderPath, this.importDataActions, this.database, this.config, this.setupOptions.shouldWriteFile);
-            await dataLoader.start(db.$id);
-            await this.importCollections(db, dataLoader);
-            await resolveAndUpdateRelationships(db.$id, this.database, this.config);
-            await this.executePostImportActions(db.$id, dataLoader);
+            if (!databaseRan) {
+                databaseRan = db;
+                dataLoader = new DataLoader(this.appwriteFolderPath, this.importDataActions, this.database, this.config, this.setupOptions.shouldWriteFile);
+                await dataLoader.start(db.$id);
+                await this.importCollections(db, dataLoader);
+                await resolveAndUpdateRelationships(db.$id, this.database, this.config);
+                await this.executePostImportActions(db.$id, dataLoader);
+            }
+            else if (databaseRan.$id !== db.$id) {
+                await this.updateOthersToFinalData(databaseRan, db);
+            }
             console.log(`---------------------------------`);
             console.log(`Finished import data for database: ${db.name}`);
             console.log(`---------------------------------`);
         }
     }
+    async updateOthersToFinalData(updatedDb, targetDb) {
+        await transferDatabaseLocalToLocal(this.database, updatedDb.$id, targetDb.$id);
+        await transferStorageLocalToLocal(this.storage, `${this.config.documentBucketId}_${updatedDb.name
+            .toLowerCase()
+            .replace(" ", "")}`, `${this.config.documentBucketId}_${targetDb.name
+            .toLowerCase()
+            .replace(" ", "")}`);
+    }
     async importCollections(db, dataLoader) {
         if (!this.config.collections) {
             return;

package/dist/migrations/storage.d.ts

@@ -6,3 +6,5 @@ export declare const initOrGetBackupStorage: (storage: Storage) => Promise<Model
 export declare const initOrGetDocumentStorage: (storage: Storage, config: AppwriteConfig, dbName: string) => Promise<Models.Bucket | undefined>;
 export declare const wipeDocumentStorage: (storage: Storage, config: AppwriteConfig, dbName: string) => Promise<void>;
 export declare const backupDatabase: (database: Databases, databaseId: string, storage: Storage) => Promise<void>;
+export declare const transferStorageLocalToLocal: (storage: Storage, fromBucketId: string, toBucketId: string) => Promise<void>;
+export declare const transferStorageLocalToRemote: (localStorage: Storage, endpoint: string, projectId: string, apiKey: string, fromBucketId: string, toBucketId: string) => Promise<void>;