appwrite-utils-cli 0.9.984 → 0.9.991

package/README.md

@@ -125,6 +125,7 @@ This updated CLI ensures that developers have robust tools at their fingertips t
 
 ## Changelog
 
+- 0.9.990: Fixed `transferFilesLocalToLocal` and `remote` if a document exists with that `$id`, also fixed wipe `"all"` option also wiping the associated buckets
 - 0.9.983: Fixed `afterImportActions` not resolving
 - 0.9.981: Try fixing `tryAwaitWithRetry` to catch `522` errors from Cloudflare, they were appearing for some users, also added a 1000ms delay to `tryAwaitWithRetry`
 - 0.9.98: Fixing some import errors reported by users

package/dist/main.js

@@ -141,7 +141,7 @@ async function main() {
             collections: parsedArgv.collectionIds?.split(","),
             doBackup: parsedArgv.backup,
             wipeDatabase: parsedArgv.wipe === "all" || parsedArgv.wipe === "docs",
-            wipeDocumentStorage: parsedArgv.wipe === "all",
+            wipeDocumentStorage: parsedArgv.wipe === "all" || parsedArgv.wipe === "storage",
             wipeUsers: parsedArgv.wipe === "all" || parsedArgv.wipe === "users",
             generateSchemas: parsedArgv.generate,
             importData: parsedArgv.import,
@@ -171,19 +171,30 @@ async function main() {
             options.wipeDocumentStorage ||
             options.wipeUsers ||
             options.wipeCollections) {
-            if (options.wipeDatabase && options.databases) {
-                for (const db of options.databases) {
-                    await controller.wipeDatabase(db);
+            if (parsedArgv.wipe === "all") {
+                if (options.databases) {
+                    for (const db of options.databases) {
+                        await controller.wipeDatabase(db, true); // true to wipe associated buckets
+                    }
                 }
+                await controller.wipeUsers();
             }
-            if (options.wipeDocumentStorage && parsedArgv.bucketIds) {
-                for (const bucketId of parsedArgv.bucketIds.split(",")) {
-                    await controller.wipeDocumentStorage(bucketId);
+            else if (parsedArgv.wipe === "docs") {
+                if (options.databases) {
+                    for (const db of options.databases) {
+                        await controller.wipeBucketFromDatabase(db);
+                    }
+                }
+                if (parsedArgv.bucketIds) {
+                    for (const bucketId of parsedArgv.bucketIds.split(",")) {
+                        await controller.wipeDocumentStorage(bucketId);
+                    }
                 }
             }
-            if (options.wipeUsers) {
+            else if (parsedArgv.wipe === "users") {
                 await controller.wipeUsers();
             }
+            // Handle specific collection wipes
             if (options.wipeCollections && options.databases) {
                 for (const db of options.databases) {
                     const dbCollections = await fetchAllCollections(db.$id, controller.database);

package/dist/migrations/importController.js

@@ -75,12 +75,26 @@ export class ImportController {
             // Find the corresponding database configs
             const updatedDbConfig = this.config.databases.find((db) => db.$id === updatedDb.$id);
             const targetDbConfig = this.config.databases.find((db) => db.$id === targetDb.$id);
-            const sourceBucketId = updatedDbConfig?.bucket?.$id ||
-                (this.config.documentBucketId &&
-                    `${this.config.documentBucketId}_${updatedDb.$id.toLowerCase().trim().replace(" ", "")}`);
-            const targetBucketId = targetDbConfig?.bucket?.$id ||
-                (this.config.documentBucketId &&
-                    `${this.config.documentBucketId}_${targetDb.$id.toLowerCase().trim().replace(" ", "")}`);
+            const allBuckets = await this.storage.listBuckets([Query.limit(1000)]);
+            const bucketsWithDbIdInThem = allBuckets.buckets.filter(bucket => bucket.name.toLowerCase().includes(updatedDb.$id.toLowerCase()));
+            const configuredUpdatedBucketId = `${this.config.documentBucketId}_${updatedDb.$id.toLowerCase().trim().replace(" ", "")}`;
+            const configuredTargetBucketId = `${this.config.documentBucketId}_${targetDb.$id.toLowerCase().trim().replace(" ", "")}`;
+            let sourceBucketId;
+            let targetBucketId;
+            if (bucketsWithDbIdInThem.find(bucket => bucket.$id === configuredUpdatedBucketId)) {
+                sourceBucketId = configuredUpdatedBucketId;
+            }
+            else if (bucketsWithDbIdInThem.find(bucket => bucket.$id === configuredTargetBucketId)) {
+                targetBucketId = configuredTargetBucketId;
+            }
+            if (!sourceBucketId) {
+                sourceBucketId = updatedDbConfig?.bucket?.$id ||
+                    bucketsWithDbIdInThem[0]?.$id;
+            }
+            if (!targetBucketId) {
+                targetBucketId = targetDbConfig?.bucket?.$id ||
+                    bucketsWithDbIdInThem[0]?.$id;
+            }
             if (sourceBucketId && targetBucketId) {
                 await transferStorageLocalToLocal(this.storage, sourceBucketId, targetBucketId);
             }

package/dist/migrations/transfer.js

@@ -33,7 +33,13 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
             }
             const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
             console.log(`Creating file: ${file.name}`);
-            tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
+            try {
+                await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
+            }
+            catch (error) {
+                // File already exists, so we can skip it
+                continue;
+            }
             numberOfFiles++;
         }
     }
@@ -59,7 +65,13 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
                 continue;
             }
             const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
-            await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
+            try {
+                await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
+            }
+            catch (error) {
+                // File already exists, so we can skip it
+                continue;
+            }
             numberOfFiles++;
         }
     }
@@ -90,7 +102,13 @@ export const transferStorageLocalToRemote = async (localStorage, endpoint, proje
     for (const file of allFromFiles) {
         const fileData = await tryAwaitWithRetry(async () => await localStorage.getFileDownload(file.bucketId, file.$id));
         const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
-        await tryAwaitWithRetry(async () => await remoteStorage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
+        try {
+            await tryAwaitWithRetry(async () => await remoteStorage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
+        }
+        catch (error) {
+            // File already exists, so we can skip it
+            continue;
+        }
         numberOfFiles++;
     }
     console.log(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`);

package/dist/utilsController.d.ts

@@ -36,10 +36,8 @@ export declare class UtilsController {
     wipeOtherDatabases(databasesToKeep: Models.Database[]): Promise<void>;
     wipeUsers(): Promise<void>;
     backupDatabase(database: Models.Database): Promise<void>;
-    wipeDatabase(database: Models.Database): Promise<{
-        collectionId: string;
-        collectionName: string;
-    }[]>;
+    wipeDatabase(database: Models.Database, wipeBucket?: boolean): Promise<void>;
+    wipeBucketFromDatabase(database: Models.Database): Promise<void>;
     wipeCollection(database: Models.Database, collection: Models.Collection): Promise<void>;
     wipeDocumentStorage(bucketId: string): Promise<void>;
     createOrUpdateCollectionsForDatabases(databases: Models.Database[], collections?: Models.Collection[]): Promise<void>;

package/dist/utilsController.js

@@ -123,11 +123,34 @@ export class UtilsController {
             throw new Error("Database, storage, or config not initialized");
         await backupDatabase(this.config, this.database, database.$id, this.storage);
     }
-    async wipeDatabase(database) {
+    async wipeDatabase(database, wipeBucket = false) {
         await this.init();
         if (!this.database)
             throw new Error("Database not initialized");
-        return await wipeDatabase(this.database, database.$id);
+        await wipeDatabase(this.database, database.$id);
+        if (wipeBucket) {
+            await this.wipeBucketFromDatabase(database);
+        }
+    }
+    async wipeBucketFromDatabase(database) {
+        // Check configured bucket in database config
+        const configuredBucket = this.config?.databases?.find(db => db.$id === database.$id)?.bucket;
+        if (configuredBucket?.$id) {
+            await this.wipeDocumentStorage(configuredBucket.$id);
+        }
+        // Also check for document bucket ID pattern
+        if (this.config?.documentBucketId) {
+            const documentBucketId = `${this.config.documentBucketId}_${database.$id.toLowerCase().trim().replace(/\s+/g, "")}`;
+            try {
+                await this.wipeDocumentStorage(documentBucketId);
+            }
+            catch (error) {
+                // Ignore if bucket doesn't exist
+                if (error?.type !== 'storage_bucket_not_found') {
+                    throw error;
+                }
+            }
+        }
     }
     async wipeCollection(database, collection) {
         await this.init();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.9.984",
+  "version": "0.9.991",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/main.ts

@@ -16,7 +16,7 @@ interface CliOptions {
   dbIds?: string;
   collectionIds?: string;
   bucketIds?: string;
-  wipe?: "all" | "docs" | "users";
+  wipe?: "all" | "storage" | "docs" | "users";
   wipeCollections?: boolean;
   generate?: boolean;
   import?: boolean;
@@ -181,7 +181,8 @@ async function main() {
       collections: parsedArgv.collectionIds?.split(","),
       doBackup: parsedArgv.backup,
       wipeDatabase: parsedArgv.wipe === "all" || parsedArgv.wipe === "docs",
-      wipeDocumentStorage: parsedArgv.wipe === "all",
+      wipeDocumentStorage:
+        parsedArgv.wipe === "all" || parsedArgv.wipe === "storage",
       wipeUsers: parsedArgv.wipe === "all" || parsedArgv.wipe === "users",
       generateSchemas: parsedArgv.generate,
       importData: parsedArgv.import,
@@ -218,19 +219,29 @@ async function main() {
       options.wipeUsers ||
       options.wipeCollections
     ) {
-      if (options.wipeDatabase && options.databases) {
-        for (const db of options.databases) {
-          await controller.wipeDatabase(db);
+      if (parsedArgv.wipe === "all") {
+        if (options.databases) {
+          for (const db of options.databases) {
+            await controller.wipeDatabase(db, true); // true to wipe associated buckets
+          }
         }
-      }
-      if (options.wipeDocumentStorage && parsedArgv.bucketIds) {
-        for (const bucketId of parsedArgv.bucketIds.split(",")) {
-          await controller.wipeDocumentStorage(bucketId);
+        await controller.wipeUsers();
+      } else if (parsedArgv.wipe === "docs") {
+        if (options.databases) {
+          for (const db of options.databases) {
+            await controller.wipeBucketFromDatabase(db);
+          }
         }
-      }
-      if (options.wipeUsers) {
+        if (parsedArgv.bucketIds) {
+          for (const bucketId of parsedArgv.bucketIds.split(",")) {
+            await controller.wipeDocumentStorage(bucketId);
+          }
+        }
+      } else if (parsedArgv.wipe === "users") {
         await controller.wipeUsers();
       }
+
+      // Handle specific collection wipes
       if (options.wipeCollections && options.databases) {
         for (const db of options.databases) {
           const dbCollections = await fetchAllCollections(

package/src/migrations/importController.ts

@@ -136,13 +136,29 @@ export class ImportController {
         (db) => db.$id === targetDb.$id
       );
 
-      const sourceBucketId = updatedDbConfig?.bucket?.$id || 
-        (this.config.documentBucketId && 
-          `${this.config.documentBucketId}_${updatedDb.$id.toLowerCase().trim().replace(" ", "")}`);
-      
-      const targetBucketId = targetDbConfig?.bucket?.$id || 
-        (this.config.documentBucketId && 
-          `${this.config.documentBucketId}_${targetDb.$id.toLowerCase().trim().replace(" ", "")}`);
+      const allBuckets = await this.storage.listBuckets([Query.limit(1000)]);
+      const bucketsWithDbIdInThem = allBuckets.buckets.filter(bucket => bucket.name.toLowerCase().includes(updatedDb.$id.toLowerCase()));
+      const configuredUpdatedBucketId = `${this.config.documentBucketId}_${updatedDb.$id.toLowerCase().trim().replace(" ", "")}`;
+      const configuredTargetBucketId = `${this.config.documentBucketId}_${targetDb.$id.toLowerCase().trim().replace(" ", "")}`;
+
+      let sourceBucketId: string | undefined;
+      let targetBucketId: string | undefined;
+
+      if (bucketsWithDbIdInThem.find(bucket => bucket.$id === configuredUpdatedBucketId)) {
+        sourceBucketId = configuredUpdatedBucketId;
+      } else if (bucketsWithDbIdInThem.find(bucket => bucket.$id === configuredTargetBucketId)) {
+        targetBucketId = configuredTargetBucketId;
+      }
+
+      if (!sourceBucketId) {
+        sourceBucketId = updatedDbConfig?.bucket?.$id ||
+          bucketsWithDbIdInThem[0]?.$id;
+      }
+
+      if (!targetBucketId) {
+        targetBucketId = targetDbConfig?.bucket?.$id ||
+          bucketsWithDbIdInThem[0]?.$id;
+      }
 
       if (sourceBucketId && targetBucketId) {
         await transferStorageLocalToLocal(

package/src/migrations/transfer.ts

@@ -64,15 +64,20 @@ export const transferStorageLocalToLocal = async (
         file.name
       );
       console.log(`Creating file: ${file.name}`);
-      tryAwaitWithRetry(
-        async () =>
+      try {
+        await tryAwaitWithRetry(
+          async () =>
           await storage.createFile(
             toBucketId,
             file.$id,
             fileToCreate,
             file.$permissions
           )
-      );
+        );
+      } catch (error: any) {
+        // File already exists, so we can skip it
+        continue;
+      }
       numberOfFiles++;
     }
   } else {
@@ -104,15 +109,20 @@ export const transferStorageLocalToLocal = async (
         new Uint8Array(fileData),
         file.name
       );
-      await tryAwaitWithRetry(
-        async () =>
+      try {
+        await tryAwaitWithRetry(
+          async () =>
           await storage.createFile(
             toBucketId,
             file.$id,
             fileToCreate,
             file.$permissions
           )
-      );
+        );
+      } catch (error: any) {
+        // File already exists, so we can skip it
+        continue;
+      }
       numberOfFiles++;
     }
   }
@@ -167,15 +177,20 @@ export const transferStorageLocalToRemote = async (
       new Uint8Array(fileData),
       file.name
     );
-    await tryAwaitWithRetry(
-      async () =>
+    try {
+      await tryAwaitWithRetry(
+        async () =>
         await remoteStorage.createFile(
           toBucketId,
           file.$id,
           fileToCreate,
           file.$permissions
         )
-    );
+      );
+    } catch (error: any) {
+      // File already exists, so we can skip it
+      continue;
+    }
     numberOfFiles++;
   }
   console.log(

package/src/utilsController.ts

@@ -179,10 +179,34 @@ export class UtilsController {
     );
   }
 
-  async wipeDatabase(database: Models.Database) {
+  async wipeDatabase(database: Models.Database, wipeBucket: boolean = false) {
     await this.init();
     if (!this.database) throw new Error("Database not initialized");
-    return await wipeDatabase(this.database, database.$id);
+    await wipeDatabase(this.database, database.$id);
+    if (wipeBucket) {
+      await this.wipeBucketFromDatabase(database);
+    }
+  }
+
+  async wipeBucketFromDatabase(database: Models.Database) {
+    // Check configured bucket in database config
+    const configuredBucket = this.config?.databases?.find(db => db.$id === database.$id)?.bucket;
+    if (configuredBucket?.$id) {
+      await this.wipeDocumentStorage(configuredBucket.$id);
+    }
+
+    // Also check for document bucket ID pattern
+    if (this.config?.documentBucketId) {
+      const documentBucketId = `${this.config.documentBucketId}_${database.$id.toLowerCase().trim().replace(/\s+/g, "")}`;
+      try {
+        await this.wipeDocumentStorage(documentBucketId);
+      } catch (error: any) {
+        // Ignore if bucket doesn't exist
+        if (error?.type !== 'storage_bucket_not_found') {
+          throw error;
+        }
+      }
+    }
   }
 
   async wipeCollection(database: Models.Database, collection: Models.Collection) {