appwrite-utils-cli 0.0.286 → 0.9.0

package/dist/migrations/openapi.js

@@ -10,36 +10,51 @@ export const generateOpenApi = async (config) => {
     for (const collection of config.collections) {
         // Transform and register each attribute schema
         const attributeSchemas = collection.attributes.map((attribute) => {
-            return transformTypeToOpenApi(attributeSchema);
+            return transformTypeToOpenApi(attributeSchema, attribute.description);
         });
         // Create and register the collection schema with descriptions
         const updatedCollectionSchema = CollectionSchema.extend({
             // @ts-ignore
-            attributes: z.array(z.union(attributeSchemas)),
+            attributes: attributeSchemas,
         }).openapi(collection.description ?? "No description");
         // Register the updated collection schema under the collection name
         registry.register(collection.name, updatedCollectionSchema);
     }
     // Convert the registry to OpenAPI JSON
-    // @ts-ignore
-    const openApiSpec = registry.toOpenAPI();
+    const generator = new OpenApiGeneratorV31(registry.definitions);
+    const openApiSpec = generator.generateComponents();
     // Output the OpenAPI spec to a file
     writeFileSync("./appwrite/openapi/openapi.json", JSON.stringify(openApiSpec, null, 2));
 };
-export function transformTypeToOpenApi(schema) {
-    return schema.transform((data) => {
-        let finalData = data;
-        if (data._def.attributes) {
-            finalData._def.attributes = data._def.attributes.map((attribute) => {
-                if (attribute.description) {
-                    return attribute.openapi(attribute.description);
+export function transformTypeToOpenApi(schema, description) {
+    // Check if description is an object (OpenAPI properties) or a string
+    let updatedSchema;
+    if (!description) {
+        return schema;
+    }
+    if (typeof description === "string") {
+        updatedSchema = schema.openapi(description);
+    }
+    else if (typeof description === "object") {
+        updatedSchema = schema.openapi(description);
+    }
+    else {
+        updatedSchema = schema;
+    }
+    // Check and transform attributes if they exist
+    if (schema._def && schema._def.shape) {
+        const shape = schema._def.shape();
+        for (const key in shape) {
+            const attributeDesc = shape[key].description;
+            if (attributeDesc) {
+                if (typeof attributeDesc === "string") {
+                    shape[key] = shape[key].openapi(attributeDesc);
                 }
-                return attribute;
-            });
-        }
-        if (schema.description) {
-            finalData.openapi(schema.description);
+                else if (typeof attributeDesc === "object") {
+                    shape[key] = shape[key].openapi(attributeDesc);
+                }
+            }
         }
-        return finalData;
-    });
+    }
+    return updatedSchema;
 }

package/dist/migrations/queue.js

@@ -2,6 +2,7 @@ import { Query } from "node-appwrite";
 import { createOrUpdateAttribute } from "./attributes.js";
 import _ from "lodash";
 import { fetchAndCacheCollectionByName } from "./collections.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export const queuedOperations = [];
 export const nameToIdMapping = new Map();
 export const enqueueOperation = (operation) => {
@@ -24,7 +25,7 @@ export const processQueue = async (db, dbId) => {
                 if (operation.collectionId) {
                     console.log(`\tFetching collection by ID: ${operation.collectionId}`);
                     try {
-                        collectionFound = await db.getCollection(dbId, operation.collectionId);
+                        collectionFound = await tryAwaitWithRetry(async () => await db.getCollection(dbId, operation.collectionId));
                     }
                     catch (e) {
                         console.log(`\tCollection not found by ID: ${operation.collectionId}`);
@@ -47,7 +48,7 @@ export const processQueue = async (db, dbId) => {
                 // Handle non-relationship operations with a specified collectionId
                 console.log(`\tFetching collection by ID: ${operation.collectionId}`);
                 try {
-                    collectionFound = await db.getCollection(dbId, operation.collectionId);
+                    collectionFound = await tryAwaitWithRetry(async () => await db.getCollection(dbId, operation.collectionId));
                 }
                 catch (e) {
                     console.log(`\tCollection not found by ID: ${operation.collectionId}`);

package/dist/migrations/relationships.d.ts

@@ -12,10 +12,10 @@ export declare const findCollectionsWithRelationships: (config: AppwriteConfig)
     twoWayKey: string;
     onDelete: "setNull" | "cascade" | "restrict";
     side: "parent" | "child";
+    description?: string | Record<string, string> | null | undefined;
+    required?: boolean | undefined;
     array?: boolean | undefined;
     error?: string | undefined;
-    required?: boolean | undefined;
-    description?: string | Record<string, string> | null | undefined;
     importMapping?: {
         originalIdField: string;
         targetField?: string | undefined;

package/dist/migrations/schemaStrings.js

@@ -17,25 +17,24 @@ export class SchemaGenerator {
         const collections = this.config.collections;
         delete this.config.collections;
         const configPath = path.join(this.appwriteFolderPath, "appwriteConfig.ts");
-        const configContent = `import { type AppwriteConfig } from "appwrite-utils";
-  
-  const appwriteConfig: AppwriteConfig = {
-    appwriteEndpoint: "${this.config.appwriteEndpoint}",
-    appwriteProject: "${this.config.appwriteProject}",
-    appwriteKey: "${this.config.appwriteKey}",
-    enableDevDatabase: ${this.config.enableDevDatabase},
-    enableBackups: ${this.config.enableBackups},
-    backupInterval: ${this.config.backupInterval},
-    backupRetention: ${this.config.backupRetention},
-    enableBackupCleanup: ${this.config.enableBackupCleanup},
-    enableMockData: ${this.config.enableMockData},
-    enableWipeOtherDatabases: ${this.config.enableWipeOtherDatabases},
-    documentBucketId: "${this.config.documentBucketId}",
-    usersCollectionName: "${this.config.usersCollectionName}",
-    databases: ${JSON.stringify(this.config.databases)}
-  };
-  
-  export default appwriteConfig;
+        const configContent = `import { type AppwriteConfig } from "appwrite-utils";
+  
+  const appwriteConfig: AppwriteConfig = {
+    appwriteEndpoint: "${this.config.appwriteEndpoint}",
+    appwriteProject: "${this.config.appwriteProject}",
+    appwriteKey: "${this.config.appwriteKey}",
+    enableBackups: ${this.config.enableBackups},
+    backupInterval: ${this.config.backupInterval},
+    backupRetention: ${this.config.backupRetention},
+    enableBackupCleanup: ${this.config.enableBackupCleanup},
+    enableMockData: ${this.config.enableMockData},
+    documentBucketId: "${this.config.documentBucketId}",
+    usersCollectionName: "${this.config.usersCollectionName}",
+    databases: ${JSON.stringify(this.config.databases)},
+    buckets: ${JSON.stringify(this.config.buckets)}
+  };
+  
+  export default appwriteConfig;
   `;
         fs.writeFileSync(configPath, configContent, { encoding: "utf-8" });
         const collectionsFolderPath = path.join(this.appwriteFolderPath, "collections");
@@ -45,19 +44,19 @@ export class SchemaGenerator {
         collections?.forEach((collection) => {
             const { databaseId, ...collectionWithoutDbId } = collection; // Destructure to exclude databaseId
             const collectionFilePath = path.join(collectionsFolderPath, `${collection.name}.ts`);
-            const collectionContent = `import { type CollectionCreate } from "appwrite-utils";
-  
-  const ${collection.name}Config: Partial<CollectionCreate> = {
-    name: "${collection.name}",
-    $id: "${collection.$id}",
-    enabled: ${collection.enabled},
-    documentSecurity: ${collection.documentSecurity},
-    $permissions: [
+            const collectionContent = `import { type CollectionCreate } from "appwrite-utils";
+  
+  const ${collection.name}Config: Partial<CollectionCreate> = {
+    name: "${collection.name}",
+    $id: "${collection.$id}",
+    enabled: ${collection.enabled},
+    documentSecurity: ${collection.documentSecurity},
+    $permissions: [
       ${collection.$permissions
                 .map((permission) => `{ permission: "${permission.permission}", target: "${permission.target}" }`)
-                .join(",\n    ")}
-    ],
-    attributes: [
+                .join(",\n    ")}
+    ],
+    attributes: [
       ${collection.attributes
                 .map((attr) => {
                 return `{ ${Object.entries(attr)
@@ -85,23 +84,21 @@ export class SchemaGenerator {
                 })
                     .join(", ")} }`;
             })
-                .join(",\n    ")}
-    ],
-    indexes: [
+                .join(",\n    ")}
+    ],
+    indexes: [
       ${(collection.indexes?.map((index) => {
                 // Map each attribute to ensure it is properly quoted
-                const formattedAttributes = index.attributes
-                    .map((attr) => `"${attr}"`)
-                    .join(", ");
+                const formattedAttributes = index.attributes.map((attr) => `"${attr}"`).join(", ") ?? "";
                 return `{ key: "${index.key}", type: "${index.type}", attributes: [${formattedAttributes}], orders: [${index.orders
                     ?.filter((order) => order !== null)
                     .map((order) => `"${order}"`)
                     .join(", ") ?? ""}] }`;
-            }) ?? []).join(",\n    ")}
-    ]
-  };
-  
-  export default ${collection.name}Config;
+            }) ?? []).join(",\n    ")}
+    ]
+  };
+  
+  export default ${collection.name}Config;
   `;
             fs.writeFileSync(collectionFilePath, collectionContent, {
                 encoding: "utf-8",
@@ -182,6 +179,11 @@ export class SchemaGenerator {
     createSchemaString = (name, attributes) => {
         const pascalName = toPascalCase(name);
         let imports = `import { z } from "zod";\n`;
+        const hasDescription = attributes.some((attr) => attr.description);
+        if (hasDescription) {
+            imports += `import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi";\n`;
+            imports += `extendZodWithOpenApi(z);\n`;
+        }
         // Use the relationshipMap to find related collections
         const relationshipDetails = this.relationshipMap.get(name) || [];
         const relatedCollections = relationshipDetails
@@ -386,6 +388,16 @@ export class SchemaGenerator {
         if (attribute.array && !attribute.required) {
             baseSchemaCode += ".nullish()";
         }
+        if (attribute.description) {
+            if (typeof attribute.description === "string") {
+                baseSchemaCode += `.openapi({ description: "${attribute.description}" })`;
+            }
+            else {
+                baseSchemaCode += `.openapi(${Object.entries(attribute.description)
+                    .map(([key, value]) => `"${key}": ${value}`)
+                    .join(", ")})`;
+            }
+        }
         return baseSchemaCode;
     };
 }

package/dist/migrations/setupDatabase.d.ts

@@ -1,7 +1,5 @@
-import { Databases, Storage } from "node-appwrite";
+import { Databases, type Models } from "node-appwrite";
 import { type AppwriteConfig } from "appwrite-utils";
-import type { SetupOptions } from "../utilsController.js";
 export declare const setupMigrationDatabase: (config: AppwriteConfig) => Promise<void>;
 export declare const ensureDatabasesExist: (config: AppwriteConfig) => Promise<void>;
-export declare const wipeOtherDatabases: (database: Databases, config: AppwriteConfig) => Promise<void>;
-export declare const startSetup: (database: Databases, storage: Storage, config: AppwriteConfig, setupOptions: SetupOptions, appwriteFolderPath: string) => Promise<void>;
+export declare const wipeOtherDatabases: (database: Databases, databasesToKeep: Models.Database[]) => Promise<void>;

package/dist/migrations/setupDatabase.js

@@ -1,49 +1,33 @@
-import { Databases, ID, Query, Storage } from "node-appwrite";
+import { Databases, Query } from "node-appwrite";
 import { createOrUpdateAttribute } from "./attributes.js";
-import { createOrUpdateCollections, generateSchemas, wipeDatabase, } from "./collections.js";
 import { getMigrationCollectionSchemas } from "./backup.js";
-import { areCollectionNamesSame, toCamelCase } from "../utils/index.js";
-import { backupDatabase, initOrGetBackupStorage, initOrGetDocumentStorage, wipeDocumentStorage, } from "./storage.js";
+import { areCollectionNamesSame, toCamelCase, tryAwaitWithRetry, } from "../utils/index.js";
 import {} from "appwrite-utils";
-import { nameToIdMapping } from "./queue.js";
-import { UsersController } from "./users.js";
+import { ulid } from "ulidx";
 export const setupMigrationDatabase = async (config) => {
-    // Create the migrations database if needed
     console.log("---------------------------------");
     console.log("Starting Migrations Setup");
     console.log("---------------------------------");
     const database = new Databases(config.appwriteClient);
     let db = null;
-    const dbCollections = [];
     const migrationCollectionsSetup = getMigrationCollectionSchemas();
     try {
-        db = await database.get("migrations");
+        db = await tryAwaitWithRetry(async () => await database.get("migrations"), undefined, true);
         console.log("Migrations database found");
     }
     catch (e) {
-        db = await database.create("migrations", "Migrations", true);
+        db = await tryAwaitWithRetry(async () => await database.create("migrations", "Migrations", true));
         console.log("Migrations database created");
     }
-    if (db) {
-        const collectionsPulled = await database.listCollections(db.$id, [
-            Query.limit(25),
-        ]);
-        dbCollections.push(...collectionsPulled.collections);
-    }
-    console.log(`Collections in migrations database: ${dbCollections.length}`);
-    // Iterate over each key in the migrationCollectionsSetup object
     for (const [collectionName, { collection, attributes }] of Object.entries(migrationCollectionsSetup)) {
-        const collectionId = toCamelCase(collectionName); // Convert name to toCamelCase for the ID
+        const collectionId = toCamelCase(collectionName);
         let collectionFound = null;
         try {
-            collectionFound = await database.getCollection(db.$id, collectionId);
+            collectionFound = await tryAwaitWithRetry(async () => await database.getCollection(db.$id, collectionId));
         }
         catch (e) {
             console.log(`Collection not found: ${collectionId}`);
-        }
-        if (!collectionFound) {
-            // Create the collection with the provided configuration
-            collectionFound = await database.createCollection(db.$id, collectionId, collectionName, undefined, collection.documentSecurity, collection.enabled);
+            collectionFound = await tryAwaitWithRetry(async () => await database.createCollection(db.$id, collectionId, collectionName, undefined, collection.documentSecurity, collection.enabled));
         }
         for (const attribute of attributes) {
             await createOrUpdateAttribute(database, db.$id, collectionFound, attribute);
@@ -56,96 +40,31 @@ export const setupMigrationDatabase = async (config) => {
 export const ensureDatabasesExist = async (config) => {
     const database = new Databases(config.appwriteClient);
     const databasesToEnsure = config.databases;
-    databasesToEnsure.push({
-        $id: "migrations",
-        name: "Migrations",
-    });
-    const dbNames = databasesToEnsure.map((db) => db.name);
-    const existingDatabases = await database.list([Query.equal("name", dbNames)]);
+    const existingDatabases = await tryAwaitWithRetry(async () => await database.list([Query.limit(500)]));
+    const migrationsDatabase = existingDatabases.databases.find((d) => d.name.toLowerCase().trim().replace(" ", "") === "migrations");
+    if (existingDatabases.databases.length !== 0 && migrationsDatabase) {
+        console.log("Wiping all databases except migrations");
+        databasesToEnsure.push(migrationsDatabase);
+    }
     for (const db of databasesToEnsure) {
         if (!existingDatabases.databases.some((d) => d.name === db.name)) {
-            await database.create(db.$id || ID.unique(), db.name, true);
+            await tryAwaitWithRetry(async () => await database.create(db.$id || ulid(), db.name, true));
             console.log(`${db.name} database created`);
         }
     }
 };
-export const wipeOtherDatabases = async (database, config) => {
-    const databasesToKeep = config.databases.map((db) => db.name.toLowerCase().trim().replace(" ", ""));
-    databasesToKeep.push("migrations");
+export const wipeOtherDatabases = async (database, databasesToKeep) => {
     console.log(`Databases to keep: ${databasesToKeep.join(", ")}`);
-    const allDatabases = await database.list([Query.limit(500)]);
+    const allDatabases = await tryAwaitWithRetry(async () => await database.list([Query.limit(500)]));
+    const migrationsDatabase = allDatabases.databases.find((d) => d.name.toLowerCase().trim().replace(" ", "") === "migrations");
+    if (allDatabases.databases.length !== 0 && migrationsDatabase) {
+        console.log("Wiping all databases except migrations");
+        databasesToKeep.push(migrationsDatabase);
+    }
     for (const db of allDatabases.databases) {
-        if (!databasesToKeep.includes(db.name.toLowerCase().trim().replace(" ", ""))) {
-            await database.delete(db.$id);
+        if (!databasesToKeep.some((d) => d.name === db.name)) {
+            await tryAwaitWithRetry(async () => await database.delete(db.$id));
             console.log(`Deleted database: ${db.name}`);
         }
     }
 };
-export const startSetup = async (database, storage, config, setupOptions, appwriteFolderPath) => {
-    await setupMigrationDatabase(config);
-    if (config.enableBackups) {
-        await initOrGetBackupStorage(storage);
-        if (setupOptions.wipeDocumentStorage) {
-            if (setupOptions.runProd) {
-                await initOrGetDocumentStorage(storage, config, config.databases[0].name);
-                await wipeDocumentStorage(storage, config, config.databases[0].name);
-            }
-            if (setupOptions.runStaging) {
-                await initOrGetDocumentStorage(storage, config, config.databases[1].name);
-                await wipeDocumentStorage(storage, config, config.databases[1].name);
-            }
-            if (setupOptions.runDev) {
-                await initOrGetDocumentStorage(storage, config, config.databases[2].name);
-                await wipeDocumentStorage(storage, config, config.databases[2].name);
-            }
-        }
-    }
-    if (config.enableWipeOtherDatabases) {
-        await wipeOtherDatabases(database, config);
-    }
-    if (setupOptions.wipeUsers) {
-        const usersController = new UsersController(config, database);
-        console.log("Wiping users");
-        await usersController.wipeUsers();
-        console.log("Users wiped");
-    }
-    await ensureDatabasesExist(config);
-    const databaseNames = config.databases.map((db) => db.name);
-    // Move to here so it always runs if it's set to true
-    if (setupOptions.generateSchemas) {
-        await generateSchemas(config, appwriteFolderPath);
-    }
-    for (const db of config.databases) {
-        // Determine if the current database should be processed based on the setup options
-        const processDatabase = (setupOptions.runProd &&
-            areCollectionNamesSame(db.name, databaseNames[0])) ||
-            (setupOptions.runStaging &&
-                areCollectionNamesSame(db.name, databaseNames[1])) ||
-            (setupOptions.runDev &&
-                areCollectionNamesSame(db.name, databaseNames[2]));
-        if (!processDatabase) {
-            continue;
-        }
-        else {
-            await initOrGetDocumentStorage(storage, config, db.name);
-        }
-        console.log(`---------------------------------`);
-        console.log(`Starting setup for database: ${db.name}`);
-        console.log(`---------------------------------`);
-        let deletedCollections;
-        if (setupOptions.wipeDatabases && processDatabase) {
-            if (config.enableBackups && setupOptions.doBackup) {
-                await backupDatabase(database, db.$id, storage);
-            }
-            deletedCollections = await wipeDatabase(database, db.$id);
-        }
-        if (processDatabase) {
-            await createOrUpdateCollections(database, db.$id, config, deletedCollections);
-        }
-        deletedCollections = undefined;
-        nameToIdMapping.clear();
-        console.log(`---------------------------------`);
-        console.log(`Finished setup for database: ${db.name}`);
-        console.log(`---------------------------------`);
-    }
-};

package/dist/migrations/storage.d.ts

@@ -3,6 +3,8 @@ import { type OperationCreate } from "./backup.js";
 import type { AppwriteConfig } from "appwrite-utils";
 export declare const logOperation: (db: Databases, dbId: string, operationDetails: OperationCreate, operationId?: string) => Promise<Models.Document>;
 export declare const initOrGetBackupStorage: (storage: Storage) => Promise<Models.Bucket>;
-export declare const initOrGetDocumentStorage: (storage: Storage, config: AppwriteConfig, dbName: string) => Promise<Models.Bucket | undefined>;
+export declare const initOrGetDocumentStorage: (storage: Storage, config: AppwriteConfig, dbId: string, bucketName?: string) => Promise<Models.Bucket | undefined>;
 export declare const wipeDocumentStorage: (storage: Storage, config: AppwriteConfig, dbName: string) => Promise<void>;
 export declare const backupDatabase: (database: Databases, databaseId: string, storage: Storage) => Promise<void>;
+export declare const transferStorageLocalToLocal: (storage: Storage, fromBucketId: string, toBucketId: string) => Promise<void>;
+export declare const transferStorageLocalToRemote: (localStorage: Storage, endpoint: string, projectId: string, apiKey: string, fromBucketId: string, toBucketId: string) => Promise<void>;

package/dist/migrations/storage.js

@@ -1,12 +1,14 @@
-import { Storage, Databases, Query, InputFile, ID, Permission, } from "node-appwrite";
+import { Storage, Databases, Query, ID, Permission, } from "node-appwrite";
+import { InputFile } from "node-appwrite/file";
 import {} from "./backup.js";
 import { splitIntoBatches } from "./migrationHelper.js";
+import { getAppwriteClient, tryAwaitWithRetry, } from "../utils/helperFunctions.js";
 export const logOperation = async (db, dbId, operationDetails, operationId) => {
     try {
         let operation;
         if (operationId) {
             // Update existing operation log
-            operation = await db.updateDocument("migrations", "currentOperations", operationId, operationDetails);
+            operation = await tryAwaitWithRetry(async () => await db.updateDocument("migrations", "currentOperations", operationId, operationDetails));
         }
         else {
             // Create new operation log
@@ -22,29 +24,30 @@ export const logOperation = async (db, dbId, operationDetails, operationId) => {
 };
 export const initOrGetBackupStorage = async (storage) => {
     try {
-        const backupStorage = await storage.getBucket("backupStorage");
+        const backupStorage = await tryAwaitWithRetry(async () => await storage.getBucket("backupStorage"));
         return backupStorage;
     }
     catch (e) {
         // ID backupStorage
         // Name Backups Storage
-        const backupStorage = await storage.createBucket("backupStorage", "Backups Storage");
+        const backupStorage = await tryAwaitWithRetry(async () => await storage.createBucket("backupStorage", "Backups Storage"));
         return backupStorage;
     }
 };
-export const initOrGetDocumentStorage = async (storage, config, dbName) => {
+export const initOrGetDocumentStorage = async (storage, config, dbId, bucketName) => {
     try {
-        await storage.getBucket(`${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`);
+        await tryAwaitWithRetry(async () => await storage.getBucket(bucketName ??
+            `${config.documentBucketId}_${dbId.toLowerCase().replace(" ", "")}`));
     }
     catch (e) {
         // ID documentStorage
         // Name Document Storage
-        const documentStorage = await storage.createBucket(`${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`, `Document Storage ${dbName}`, [
+        const documentStorage = await tryAwaitWithRetry(async () => await storage.createBucket(`${config.documentBucketId}_${dbId.toLowerCase().replace(" ", "")}`, `${dbId} Storage`, [
             Permission.read("any"),
             Permission.create("users"),
             Permission.update("users"),
             Permission.delete("users"),
-        ]);
+        ]));
         return documentStorage;
     }
 };
@@ -61,7 +64,7 @@ export const wipeDocumentStorage = async (storage, config, dbName) => {
         if (lastFileId) {
             queries.push(Query.cursorAfter(lastFileId));
         }
-        const filesPulled = await storage.listFiles(bucketId, queries);
+        const filesPulled = await tryAwaitWithRetry(async () => await storage.listFiles(bucketId, queries));
         if (filesPulled.files.length === 0) {
             console.log("No files found, done!");
             moreFiles = false;
@@ -78,7 +81,7 @@ export const wipeDocumentStorage = async (storage, config, dbName) => {
     }
     for (const fileId of allFiles) {
         console.log(`Deleting file: ${fileId}`);
-        await storage.deleteFile(bucketId, fileId);
+        await tryAwaitWithRetry(async () => await storage.deleteFile(bucketId, fileId));
     }
     console.log(`All files in bucket ${bucketId} have been deleted.`);
 };
@@ -124,7 +127,7 @@ export const backupDatabase = async (database, databaseId, storage) => {
     // Fetch and backup the database details
     let db;
     try {
-        db = await database.get(databaseId);
+        db = await tryAwaitWithRetry(async () => await database.get(databaseId));
     }
     catch (e) {
         console.error(`Error fetching database: ${e}`);
@@ -146,25 +149,25 @@ export const backupDatabase = async (database, databaseId, storage) => {
     let progress = 0;
     let total = 0; // Initialize total to 0, will be updated dynamically
     while (moreCollections) {
-        const collectionResponse = await database.listCollections(databaseId, [
+        const collectionResponse = await tryAwaitWithRetry(async () => await database.listCollections(databaseId, [
             Query.limit(500), // Adjust the limit as needed
             ...(lastCollectionId ? [Query.cursorAfter(lastCollectionId)] : []),
-        ]);
+        ]));
         total += collectionResponse.collections.length; // Update total with number of collections
         for (const { $id: collectionId, name: collectionName, } of collectionResponse.collections) {
             let collectionDocumentCount = 0; // Initialize document count for the current collection
             try {
-                const collection = await database.getCollection(databaseId, collectionId);
+                const collection = await tryAwaitWithRetry(async () => await database.getCollection(databaseId, collectionId));
                 progress++;
                 data.collections.push(JSON.stringify(collection));
                 // Initialize pagination for documents within the current collection
                 let lastDocumentId = "";
                 let moreDocuments = true;
                 while (moreDocuments) {
-                    const documentResponse = await database.listDocuments(databaseId, collectionId, [
+                    const documentResponse = await tryAwaitWithRetry(async () => await database.listDocuments(databaseId, collectionId, [
                         Query.limit(500), // Adjust the limit as needed
                         ...(lastDocumentId ? [Query.cursorAfter(lastDocumentId)] : []),
-                    ]);
+                    ]));
                     total += documentResponse.documents.length; // Update total with number of documents
                     collectionDocumentCount += documentResponse.documents.length; // Update document count for the current collection
                     let documentPromises = [];
@@ -244,3 +247,94 @@ export const backupDatabase = async (database, databaseId, storage) => {
     console.log("Database Backup Complete");
     console.log("---------------------------------");
 };
+export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucketId) => {
+    console.log(`Transferring files from ${fromBucketId} to ${toBucketId}`);
+    let lastFileId;
+    let fromFiles = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [Query.limit(100)]));
+    const allFromFiles = fromFiles.files;
+    let numberOfFiles = 0;
+    const downloadFileWithRetry = async (bucketId, fileId) => {
+        let attempts = 3;
+        while (attempts > 0) {
+            try {
+                return await storage.getFileDownload(bucketId, fileId);
+            }
+            catch (error) {
+                console.error(`Error downloading file ${fileId}: ${error}`);
+                attempts--;
+                if (attempts === 0)
+                    throw error;
+            }
+        }
+    };
+    if (fromFiles.files.length < 100) {
+        for (const file of allFromFiles) {
+            const fileData = await tryAwaitWithRetry(async () => await downloadFileWithRetry(file.bucketId, file.$id));
+            if (!fileData) {
+                console.error(`Error downloading file ${file.$id}`);
+                continue;
+            }
+            const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
+            console.log(`Creating file: ${file.name}`);
+            tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
+            numberOfFiles++;
+        }
+    }
+    else {
+        lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
+        while (lastFileId) {
+            const files = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [
+                Query.limit(100),
+                Query.cursorAfter(lastFileId),
+            ]));
+            allFromFiles.push(...files.files);
+            if (files.files.length < 100) {
+                lastFileId = undefined;
+            }
+            else {
+                lastFileId = files.files[files.files.length - 1].$id;
+            }
+        }
+        for (const file of allFromFiles) {
+            const fileData = await tryAwaitWithRetry(async () => await downloadFileWithRetry(file.bucketId, file.$id));
+            if (!fileData) {
+                console.error(`Error downloading file ${file.$id}`);
+                continue;
+            }
+            const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
+            await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
+            numberOfFiles++;
+        }
+    }
+    console.log(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`);
+};
+export const transferStorageLocalToRemote = async (localStorage, endpoint, projectId, apiKey, fromBucketId, toBucketId) => {
+    console.log(`Transferring files from current storage ${fromBucketId} to ${endpoint} bucket ${toBucketId}`);
+    const client = getAppwriteClient(endpoint, apiKey, projectId);
+    const remoteStorage = new Storage(client);
+    let numberOfFiles = 0;
+    let lastFileId;
+    let fromFiles = await tryAwaitWithRetry(async () => await localStorage.listFiles(fromBucketId, [Query.limit(100)]));
+    const allFromFiles = fromFiles.files;
+    if (fromFiles.files.length === 100) {
+        lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
+        while (lastFileId) {
+            const files = await tryAwaitWithRetry(async () => await localStorage.listFiles(fromBucketId, [
+                Query.limit(100),
+                Query.cursorAfter(lastFileId),
+            ]));
+            allFromFiles.push(...files.files);
+            if (files.files.length < 100) {
+                break;
+            }
+            lastFileId = files.files[files.files.length - 1].$id;
+        }
+    }
+    for (const file of allFromFiles) {
+        const fileData = await tryAwaitWithRetry(async () => await localStorage.getFileDownload(file.bucketId, file.$id));
+        const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
+        await tryAwaitWithRetry(async () => await remoteStorage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
+        numberOfFiles++;
+    }
+    console.log(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`);
+};