appwrite-utils-cli 0.9.91 → 0.9.93

package/README.md

@@ -125,6 +125,8 @@ This updated CLI ensures that developers have robust tools at their fingertips t
 
 ## Changelog
 
+- 0.9.93: Updated `selectDatabases` and `selectCollections` from the interactive CLI to prefer local collections or databases when synchronizing the databases
+- 0.9.92: Fixed `createOrUpdateAttributes` so it deletes attributes that don't exist in local config when you are running `syncDb`. Also updated the database and collection selection, so it won't try and fetch the collections and databases that don't exist (ones you picked from local config) and error
 - 0.9.91: Fixed another webpack error, screw you react (but you're supported now so I guess not-screw-you)
 - 0.9.90: Fixed Webpack errors (why tf does webpack add an extra `default`...???)
 - 0.9.80: Fixed collections not being unique between local and remote

package/dist/collections/attributes.js

@@ -232,10 +232,28 @@ export const createOrUpdateAttribute = async (db, dbId, collection, attribute) =
 };
 export const createUpdateCollectionAttributes = async (db, dbId, collection, attributes) => {
     console.log(chalk.green(`Creating/Updating attributes for collection: ${collection.name}`));
+    // @ts-expect-error
+    const existingAttributes = collection.attributes.map((attr) => parseAttribute(attr)) || [];
+    const attributesToRemove = existingAttributes.filter((attr) => !attributes.some((a) => a.key === attr.key));
+    const indexesToRemove = collection.indexes.filter((index) => attributesToRemove.some((attr) => index.attributes.includes(attr.key)));
+    if (attributesToRemove.length > 0) {
+        if (indexesToRemove.length > 0) {
+            console.log(chalk.red(`Removing indexes as they rely on an attribute that is being removed: ${indexesToRemove.map((index) => index.key).join(", ")}`));
+            for (const index of indexesToRemove) {
+                await tryAwaitWithRetry(async () => await db.deleteIndex(dbId, collection.$id, index.key));
+                await delay(100);
+            }
+        }
+        for (const attr of attributesToRemove) {
+            console.log(chalk.red(`Removing attribute: ${attr.key} as it is no longer in the collection`));
+            await tryAwaitWithRetry(async () => await db.deleteAttribute(dbId, collection.$id, attr.key));
+            await delay(50);
+        }
+    }
     const batchSize = 3;
     for (let i = 0; i < attributes.length; i += batchSize) {
         const batch = attributes.slice(i, i + batchSize);
-        const attributePromises = batch.map((attribute) => createOrUpdateAttribute(db, dbId, collection, attribute));
+        const attributePromises = batch.map((attribute) => tryAwaitWithRetry(async () => await createOrUpdateAttribute(db, dbId, collection, attribute)));
         const results = await Promise.allSettled(attributePromises);
         results.forEach((result) => {
             if (result.status === "rejected") {
@@ -243,7 +261,7 @@ export const createUpdateCollectionAttributes = async (db, dbId, collection, att
             }
         });
         // Add delay after each batch
-        await delay(500);
+        await delay(200);
     }
     console.log(`Finished creating/updating attributes for collection: ${collection.name}`);
 };

package/dist/interactiveCLI.js

@@ -4,9 +4,9 @@ import { createEmptyCollection, setupDirsFiles } from "./utils/setupFiles.js";
 import { fetchAllDatabases } from "./databases/methods.js";
 import { fetchAllCollections } from "./collections/methods.js";
 import { listBuckets, createBucket } from "./storage/methods.js";
-import { Databases, Storage, Client, Compression, } from "node-appwrite";
+import { Databases, Storage, Client, Compression, Query, } from "node-appwrite";
 import { getClient } from "./utils/getClientFromConfig.js";
-import { parseAttribute, PermissionToAppwritePermission } from "appwrite-utils";
+import { parseAttribute, PermissionToAppwritePermission, } from "appwrite-utils";
 import { ulid } from "ulidx";
 import chalk from "chalk";
 import { DateTime } from "luxon";
@@ -106,15 +106,34 @@ export class InteractiveCLI {
     async selectDatabases(databases, message, multiSelect = true) {
         await this.initControllerIfNeeded();
         const configDatabases = this.getLocalDatabases();
-        const allDatabases = [...databases, ...configDatabases].reduce((acc, db) => {
-            if (!acc.find(d => d.name === db.name)) {
+        const allDatabases = [...databases, ...configDatabases]
+            .reduce((acc, db) => {
+            // Local config takes precedence - if a database with same name exists, use local version
+            const existingIndex = acc.findIndex((d) => d.name === db.name);
+            if (existingIndex >= 0) {
+                if (configDatabases.some((cdb) => cdb.name === db.name)) {
+                    acc[existingIndex] = db; // Replace with local version
+                }
+            }
+            else {
                 acc.push(db);
             }
             return acc;
-        }, []);
+        }, [])
+            .filter((db) => db.name.toLowerCase() !== "migrations");
+        const hasLocalAndRemote = allDatabases.some((db) => configDatabases.some((c) => c.name === db.name)) &&
+            allDatabases.some((db) => !configDatabases.some((c) => c.name === db.name));
         const choices = allDatabases
             .sort((a, b) => a.name.localeCompare(b.name))
-            .map((db) => ({ name: db.name, value: db }))
+            .map((db) => ({
+            name: db.name +
+                (hasLocalAndRemote
+                    ? configDatabases.some((c) => c.name === db.name)
+                        ? " (Local)"
+                        : " (Remote)"
+                    : ""),
+            value: db,
+        }))
             .filter((db) => db.name.toLowerCase() !== "migrations");
         const { selectedDatabases } = await inquirer.prompt([
             {
@@ -128,15 +147,41 @@ export class InteractiveCLI {
         ]);
         return selectedDatabases;
     }
-    async selectCollections(database, databasesClient, message, multiSelect = true) {
+    async selectCollections(database, databasesClient, message, multiSelect = true, preferLocal = false) {
         await this.initControllerIfNeeded();
-        const collections = await fetchAllCollections(database.$id, databasesClient);
         const configCollections = this.getLocalCollections();
-        const collectionNames = collections.map((c) => c.name).concat(configCollections.map((c) => c.name));
-        const allCollectionNamesUnique = Array.from(new Set(collectionNames));
-        const allCollections = allCollectionNamesUnique.map((name) => configCollections.find((c) => c.name === name) ?? collections.find((c) => c.name === name)).filter((v) => v !== undefined);
-        const choices = allCollections.map((collection) => ({
-            name: collection.name,
+        let remoteCollections = [];
+        const dbExists = await databasesClient.list([
+            Query.equal("name", database.name),
+        ]);
+        if (dbExists.total === 0) {
+            console.log(chalk.red(`Database "${database.name}" does not exist, using only local collection options`));
+        }
+        else {
+            remoteCollections = await fetchAllCollections(database.$id, databasesClient);
+        }
+        const allCollections = preferLocal
+            ? remoteCollections.reduce((acc, remoteCollection) => {
+                if (!acc.some((c) => c.name === remoteCollection.name)) {
+                    acc.push(remoteCollection);
+                }
+                return acc;
+            }, [...configCollections])
+            : [
+                ...remoteCollections,
+                ...configCollections.filter((c) => !remoteCollections.some((rc) => rc.name === c.name)),
+            ];
+        const hasLocalAndRemote = allCollections.some((coll) => configCollections.some((c) => c.name === coll.name)) &&
+            allCollections.some((coll) => !configCollections.some((c) => c.name === coll.name));
+        const choices = allCollections
+            .sort((a, b) => a.name.localeCompare(b.name))
+            .map((collection) => ({
+            name: collection.name +
+                (hasLocalAndRemote
+                    ? configCollections.some((c) => c.name === collection.name)
+                        ? " (Local)"
+                        : " (Remote)"
+                    : ""),
             value: collection,
         }));
         const { selectedCollections } = await inquirer.prompt([
@@ -188,7 +233,14 @@ export class InteractiveCLI {
         const allBuckets = await listBuckets(storage);
         // If there are no buckets, ask to create one for each database
         if (allBuckets.total === 0) {
-            for (const database of databases ?? config.databases) {
+            const databasesToUse = databases ?? config.databases;
+            for (const database of databasesToUse) {
+                // If database has bucket config in local config, use that
+                const localDatabase = this.controller.config?.databases.find((db) => db.name === database.name);
+                if (localDatabase?.bucket) {
+                    database.bucket = localDatabase.bucket;
+                    continue;
+                }
                 const { wantCreateBucket } = await inquirer.prompt([
                     {
                         type: "confirm",
@@ -354,7 +406,8 @@ export class InteractiveCLI {
     async syncDb() {
         console.log(chalk.yellow("Syncing database..."));
         const databases = await this.selectDatabases(await fetchAllDatabases(this.controller.database), chalk.blue("Select databases to synchronize:"), true);
-        const collections = await this.selectCollections(databases[0], this.controller.database, chalk.blue("Select collections to synchronize:"), true);
+        const collections = await this.selectCollections(databases[0], this.controller.database, chalk.blue("Select collections to synchronize:"), true, true // prefer local
+        );
         await this.controller.syncDb(databases, collections);
         console.log(chalk.green("Database sync completed."));
     }
@@ -486,7 +539,7 @@ export class InteractiveCLI {
         ]);
         const options = {
             databases,
-            collections: collections.map(c => c.name),
+            collections: collections.map((c) => c.name),
             doBackup,
             importData: true,
             shouldWriteFile,
@@ -555,7 +608,8 @@ export class InteractiveCLI {
         if (!targetDb) {
             throw new Error("No target database selected");
         }
-        const selectedCollections = await this.selectCollections(fromDb, sourceClient, "Select collections to transfer:");
+        const selectedCollections = await this.selectCollections(fromDb, sourceClient, "Select collections to transfer:", true, false // don't prefer local for transfers
+        );
         const { transferStorage } = await inquirer.prompt([
             {
                 type: "confirm",
@@ -602,7 +656,7 @@ export class InteractiveCLI {
     getLocalCollections() {
         const configCollections = this.controller.config?.collections || [];
         // @ts-expect-error - appwrite invalid types
-        return configCollections.map(c => ({
+        return configCollections.map((c) => ({
             $id: c.$id || ulid(),
             $createdAt: DateTime.now().toISO(),
             $updatedAt: DateTime.now().toISO(),
@@ -617,7 +671,7 @@ export class InteractiveCLI {
     }
     getLocalDatabases() {
         const configDatabases = this.controller.config?.databases || [];
-        return configDatabases.map(db => ({
+        return configDatabases.map((db) => ({
             $id: db.$id || ulid(),
             $createdAt: DateTime.now().toISO(),
             $updatedAt: DateTime.now().toISO(),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.9.91",
+  "version": "0.9.93",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/collections/attributes.ts

@@ -471,11 +471,32 @@ export const createUpdateCollectionAttributes = async (
     chalk.green(`Creating/Updating attributes for collection: ${collection.name}`)
   );
 
+  // @ts-expect-error
+  const existingAttributes: Attribute[] = collection.attributes.map((attr) => parseAttribute(attr)) || [];
+
+  const attributesToRemove = existingAttributes.filter((attr) => !attributes.some((a) => a.key === attr.key));
+  const indexesToRemove = collection.indexes.filter((index) => attributesToRemove.some((attr) => index.attributes.includes(attr.key)));
+
+  if (attributesToRemove.length > 0) {
+    if (indexesToRemove.length > 0) {
+      console.log(chalk.red(`Removing indexes as they rely on an attribute that is being removed: ${indexesToRemove.map((index) => index.key).join(", ")}`));
+      for (const index of indexesToRemove) {
+        await tryAwaitWithRetry(async () => await db.deleteIndex(dbId, collection.$id, index.key));
+        await delay(100);
+      }
+    }
+    for (const attr of attributesToRemove) {
+      console.log(chalk.red(`Removing attribute: ${attr.key} as it is no longer in the collection`));
+      await tryAwaitWithRetry(async () => await db.deleteAttribute(dbId, collection.$id, attr.key));
+      await delay(50);
+    }
+  }
+
   const batchSize = 3;
   for (let i = 0; i < attributes.length; i += batchSize) {
     const batch = attributes.slice(i, i + batchSize);
     const attributePromises = batch.map((attribute) =>
-      createOrUpdateAttribute(db, dbId, collection, attribute)
+      tryAwaitWithRetry(async () => await createOrUpdateAttribute(db, dbId, collection, attribute))
     );
 
     const results = await Promise.allSettled(attributePromises);
@@ -486,7 +507,7 @@ export const createUpdateCollectionAttributes = async (
     });
 
     // Add delay after each batch
-    await delay(500);
+    await delay(200);
   }
   console.log(
     `Finished creating/updating attributes for collection: ${collection.name}`

package/src/interactiveCLI.ts

@@ -10,10 +10,16 @@ import {
   Client,
   type Models,
   Compression,
+  Query,
 } from "node-appwrite";
 import { getClient } from "./utils/getClientFromConfig.js";
 import type { TransferOptions } from "./migrations/transfer.js";
-import { parseAttribute, PermissionToAppwritePermission, type AppwriteConfig, type ConfigDatabases } from "appwrite-utils";
+import {
+  parseAttribute,
+  PermissionToAppwritePermission,
+  type AppwriteConfig,
+  type ConfigDatabases,
+} from "appwrite-utils";
 import { ulid } from "ulidx";
 import chalk from "chalk";
 import { DateTime } from "luxon";
@@ -40,9 +46,13 @@ export class InteractiveCLI {
   constructor(private currentDir: string) {}
 
   async run(): Promise<void> {
-    console.log(chalk.green("Welcome to Appwrite Utils CLI Tool by Zach Handley"));
     console.log(
-      chalk.blue("For more information, visit https://github.com/zachhandley/AppwriteUtils")
+      chalk.green("Welcome to Appwrite Utils CLI Tool by Zach Handley")
+    );
+    console.log(
+      chalk.blue(
+        "For more information, visit https://github.com/zachhandley/AppwriteUtils"
+      )
     );
 
     while (true) {
@@ -123,18 +133,43 @@ export class InteractiveCLI {
   ): Promise<Models.Database[]> {
     await this.initControllerIfNeeded();
     const configDatabases = this.getLocalDatabases();
-    const allDatabases = [...databases, ...configDatabases].reduce((acc, db) => {
-      if (!acc.find(d => d.name === db.name)) {
-        acc.push(db);
-      }
-      return acc;
-    }, [] as Models.Database[]);
-  
+    const allDatabases = [...databases, ...configDatabases]
+      .reduce((acc, db) => {
+        // Local config takes precedence - if a database with same name exists, use local version
+        const existingIndex = acc.findIndex((d) => d.name === db.name);
+        if (existingIndex >= 0) {
+          if (configDatabases.some((cdb) => cdb.name === db.name)) {
+            acc[existingIndex] = db; // Replace with local version
+          }
+        } else {
+          acc.push(db);
+        }
+        return acc;
+      }, [] as Models.Database[])
+      .filter((db) => db.name.toLowerCase() !== "migrations");
+
+    const hasLocalAndRemote =
+      allDatabases.some((db) =>
+        configDatabases.some((c) => c.name === db.name)
+      ) &&
+      allDatabases.some(
+        (db) => !configDatabases.some((c) => c.name === db.name)
+      );
+
     const choices = allDatabases
       .sort((a, b) => a.name.localeCompare(b.name))
-      .map((db) => ({ name: db.name, value: db }))
+      .map((db) => ({
+        name:
+          db.name +
+          (hasLocalAndRemote
+            ? configDatabases.some((c) => c.name === db.name)
+              ? " (Local)"
+              : " (Remote)"
+            : ""),
+        value: db,
+      }))
       .filter((db) => db.name.toLowerCase() !== "migrations");
-  
+
     const { selectedDatabases } = await inquirer.prompt([
       {
         type: multiSelect ? "checkbox" : "list",
@@ -145,7 +180,7 @@ export class InteractiveCLI {
         pageSize: 10,
       },
     ]);
-  
+
     return selectedDatabases;
   }
 
@@ -153,21 +188,67 @@ export class InteractiveCLI {
     database: Models.Database,
     databasesClient: Databases,
     message: string,
-    multiSelect = true 
+    multiSelect = true,
+    preferLocal = false
   ): Promise<Models.Collection[]> {
     await this.initControllerIfNeeded();
-    const collections = await fetchAllCollections(
-      database.$id,
-      databasesClient
-    );
+
     const configCollections = this.getLocalCollections();
-    const collectionNames = collections.map((c) => c.name).concat(configCollections.map((c) => c.name));
-    const allCollectionNamesUnique = Array.from(new Set(collectionNames));
-    const allCollections = allCollectionNamesUnique.map((name) => configCollections.find((c) => c.name === name) ?? collections.find((c) => c.name === name)).filter((v) => v !== undefined);
-    const choices = allCollections.map((collection) => ({
-      name: collection.name,
-      value: collection,
-    }));
+    let remoteCollections: Models.Collection[] = [];
+
+    const dbExists = await databasesClient.list([
+      Query.equal("name", database.name),
+    ]);
+    if (dbExists.total === 0) {
+      console.log(
+        chalk.red(
+          `Database "${database.name}" does not exist, using only local collection options`
+        )
+      );
+    } else {
+      remoteCollections = await fetchAllCollections(
+        database.$id,
+        databasesClient
+      );
+    }
+
+    const allCollections = preferLocal
+      ? remoteCollections.reduce(
+          (acc, remoteCollection) => {
+            if (!acc.some((c) => c.name === remoteCollection.name)) {
+              acc.push(remoteCollection);
+            }
+            return acc;
+          },
+          [...configCollections]
+        )
+      : [
+          ...remoteCollections,
+          ...configCollections.filter(
+            (c) => !remoteCollections.some((rc) => rc.name === c.name)
+          ),
+        ];
+
+    const hasLocalAndRemote =
+      allCollections.some((coll) =>
+        configCollections.some((c) => c.name === coll.name)
+      ) &&
+      allCollections.some(
+        (coll) => !configCollections.some((c) => c.name === coll.name)
+      );
+
+    const choices = allCollections
+      .sort((a, b) => a.name.localeCompare(b.name))
+      .map((collection) => ({
+        name:
+          collection.name +
+          (hasLocalAndRemote
+            ? configCollections.some((c) => c.name === collection.name)
+              ? " (Local)"
+              : " (Remote)"
+            : ""),
+        value: collection,
+      }));
 
     const { selectedCollections } = await inquirer.prompt([
       {
@@ -217,7 +298,9 @@ export class InteractiveCLI {
           input.trim() !== "" || "Collection name cannot be empty.",
       },
     ]);
-    console.log(chalk.green(`Creating collection config file for '${collectionName}'...`));
+    console.log(
+      chalk.green(`Creating collection config file for '${collectionName}'...`)
+    );
     createEmptyCollection(collectionName);
   }
 
@@ -236,12 +319,23 @@ export class InteractiveCLI {
 
     // If there are no buckets, ask to create one for each database
     if (allBuckets.total === 0) {
-      for (const database of databases ?? config.databases) {
+      const databasesToUse = databases ?? config.databases;
+      for (const database of databasesToUse) {
+        // If database has bucket config in local config, use that
+        const localDatabase = this.controller!.config?.databases.find(
+          (db) => db.name === database.name
+        );
+        if (localDatabase?.bucket) {
+          database.bucket = localDatabase.bucket;
+          continue;
+        }
         const { wantCreateBucket } = await inquirer.prompt([
           {
             type: "confirm",
             name: "wantCreateBucket",
-            message: chalk.blue(`There are no buckets. Do you want to create a bucket for the database "${database.name}"?`),
+            message: chalk.blue(
+              `There are no buckets. Do you want to create a bucket for the database "${database.name}"?`
+            ),
             default: true,
           },
         ]);
@@ -448,13 +542,14 @@ export class InteractiveCLI {
     const databases = await this.selectDatabases(
       await fetchAllDatabases(this.controller!.database!),
       chalk.blue("Select databases to synchronize:"),
-      true,
+      true
     );
     const collections = await this.selectCollections(
       databases[0],
       this.controller!.database!,
       chalk.blue("Select collections to synchronize:"),
       true,
+      true // prefer local
     );
     await this.controller!.syncDb(databases, collections);
     console.log(chalk.green("Database sync completed."));
@@ -600,13 +695,19 @@ export class InteractiveCLI {
       ]);
 
       if (confirm) {
-        console.log(chalk.yellow(`Wiping selected collections from ${database.name}...`));
+        console.log(
+          chalk.yellow(`Wiping selected collections from ${database.name}...`)
+        );
         for (const collection of collections) {
           await this.controller!.wipeCollection(database, collection);
-          console.log(chalk.green(`Collection ${collection.name} wiped successfully.`));
+          console.log(
+            chalk.green(`Collection ${collection.name} wiped successfully.`)
+          );
         }
       } else {
-        console.log(chalk.blue(`Wipe operation cancelled for ${database.name}.`));
+        console.log(
+          chalk.blue(`Wipe operation cancelled for ${database.name}.`)
+        );
       }
     }
     console.log(chalk.green("Wipe collections operation completed."));
@@ -654,7 +755,7 @@ export class InteractiveCLI {
 
     const options = {
       databases,
-      collections: collections.map(c => c.name),
+      collections: collections.map((c) => c.name),
       doBackup,
       importData: true,
       shouldWriteFile,
@@ -753,7 +854,9 @@ export class InteractiveCLI {
     const selectedCollections = await this.selectCollections(
       fromDb,
       sourceClient,
-      "Select collections to transfer:"
+      "Select collections to transfer:",
+      true,
+      false // don't prefer local for transfers
     );
 
     const { transferStorage } = await inquirer.prompt([
@@ -822,11 +925,10 @@ export class InteractiveCLI {
     console.log(chalk.green("Data transfer completed."));
   }
 
-
   private getLocalCollections(): Models.Collection[] {
     const configCollections = this.controller!.config?.collections || [];
     // @ts-expect-error - appwrite invalid types
-    return configCollections.map(c => ({
+    return configCollections.map((c) => ({
       $id: c.$id || ulid(),
       $createdAt: DateTime.now().toISO(),
       $updatedAt: DateTime.now().toISO(),
@@ -842,7 +944,7 @@ export class InteractiveCLI {
 
   private getLocalDatabases(): Models.Database[] {
     const configDatabases = this.controller!.config?.databases || [];
-    return configDatabases.map(db => ({
+    return configDatabases.map((db) => ({
       $id: db.$id || ulid(),
       $createdAt: DateTime.now().toISO(),
       $updatedAt: DateTime.now().toISO(),
@@ -860,4 +962,4 @@ export class InteractiveCLI {
       console.error(chalk.red("Error reloading configuration files:"), error);
     }
   }
-}
+}