appwrite-utils-cli 0.9.77 → 0.9.78

package/dist/utilsController.d.ts

@@ -7,6 +7,7 @@ export interface SetupOptions {
     collections?: string[];
     doBackup?: boolean;
     wipeDatabase?: boolean;
+    wipeCollections?: boolean;
     wipeDocumentStorage?: boolean;
     wipeUsers?: boolean;
     generateSchemas?: boolean;
@@ -26,9 +27,11 @@ export declare class UtilsController {
     afterImportActionsDefinitions: AfterImportActions;
     constructor(currentUserDir: string);
     init(): Promise<void>;
+    reloadConfig(): Promise<void>;
     setupMigrationDatabase(): Promise<void>;
     ensureDatabaseConfigBucketsExist(databases?: Models.Database[]): Promise<void>;
     ensureDatabasesExist(databases?: Models.Database[]): Promise<void>;
+    ensureCollectionsExist(database: Models.Database, collections?: Models.Collection[]): Promise<void>;
     getDatabasesByIds(ids: string[]): Promise<Models.Database[]>;
     wipeOtherDatabases(databasesToKeep: Models.Database[]): Promise<void>;
     wipeUsers(): Promise<void>;
@@ -37,16 +40,17 @@ export declare class UtilsController {
         collectionId: string;
         collectionName: string;
     }[]>;
+    wipeCollection(database: Models.Database, collection: Models.Collection): Promise<void>;
     wipeDocumentStorage(bucketId: string): Promise<void>;
-    createOrUpdateCollectionsForDatabases(databases: Models.Database[]): Promise<void>;
+    createOrUpdateCollectionsForDatabases(databases: Models.Database[], collections?: Models.Collection[]): Promise<void>;
     createOrUpdateCollections(database: Models.Database, deletedCollections?: {
         collectionId: string;
         collectionName: string;
-    }[]): Promise<void>;
+    }[], collections?: Models.Collection[]): Promise<void>;
     generateSchemas(): Promise<void>;
-    importData(options: SetupOptions): Promise<void>;
+    importData(options?: SetupOptions): Promise<void>;
     synchronizeConfigurations(databases?: Models.Database[], config?: AppwriteConfig): Promise<void>;
-    syncDb(): Promise<void>;
+    syncDb(databases?: Models.Database[], collections?: Models.Collection[]): Promise<void>;
     getAppwriteFolderPath(): string;
     transferData(options: TransferOptions): Promise<void>;
 }

package/dist/utilsController.js

@@ -5,8 +5,8 @@ import { UsersController } from "./migrations/users.js";
 import { AppwriteToX } from "./migrations/appwriteToX.js";
 import { ImportController } from "./migrations/importController.js";
 import { ImportDataActions } from "./migrations/importDataActions.js";
-import { setupMigrationDatabase, ensureDatabasesExist, wipeOtherDatabases, } from "./migrations/setupDatabase.js";
-import { createOrUpdateCollections, wipeDatabase, generateSchemas, fetchAllCollections, } from "./collections/methods.js";
+import { setupMigrationDatabase, ensureDatabasesExist, wipeOtherDatabases, ensureCollectionsExist, } from "./migrations/setupDatabase.js";
+import { createOrUpdateCollections, wipeDatabase, generateSchemas, fetchAllCollections, wipeCollection, } from "./collections/methods.js";
 import { backupDatabase, ensureDatabaseConfigBucketsExist, initOrGetBackupStorage, wipeDocumentStorage, } from "./storage/methods.js";
 import path from "path";
 import { converterFunctions, validationRules, } from "appwrite-utils";
@@ -53,6 +53,17 @@ export class UtilsController {
             this.config.appwriteClient = this.appwriteServer;
         }
     }
+    async reloadConfig() {
+        this.config = await loadConfig(this.appwriteFolderPath);
+        this.appwriteServer = new Client();
+        this.appwriteServer
+            .setEndpoint(this.config.appwriteEndpoint)
+            .setProject(this.config.appwriteProject)
+            .setKey(this.config.appwriteKey);
+        this.database = new Databases(this.appwriteServer);
+        this.storage = new Storage(this.appwriteServer);
+        this.config.appwriteClient = this.appwriteServer;
+    }
     async setupMigrationDatabase() {
         await this.init();
         if (!this.config)
@@ -73,7 +84,13 @@ export class UtilsController {
             throw new Error("Config not initialized");
         await this.setupMigrationDatabase();
         await this.ensureDatabaseConfigBucketsExist(databases);
-        await ensureDatabasesExist(this.config);
+        await ensureDatabasesExist(this.config, databases);
+    }
+    async ensureCollectionsExist(database, collections) {
+        await this.init();
+        if (!this.config)
+            throw new Error("Config not initialized");
+        await ensureCollectionsExist(this.config, database, collections);
     }
     async getDatabasesByIds(ids) {
         await this.init();
@@ -110,27 +127,33 @@ export class UtilsController {
             throw new Error("Database not initialized");
         return await wipeDatabase(this.database, database.$id);
     }
+    async wipeCollection(database, collection) {
+        await this.init();
+        if (!this.database)
+            throw new Error("Database not initialized");
+        await wipeCollection(this.database, database.$id, collection.$id);
+    }
     async wipeDocumentStorage(bucketId) {
         await this.init();
         if (!this.storage)
             throw new Error("Storage not initialized");
         await wipeDocumentStorage(this.storage, bucketId);
     }
-    async createOrUpdateCollectionsForDatabases(databases) {
+    async createOrUpdateCollectionsForDatabases(databases, collections = []) {
         await this.init();
         if (!this.database || !this.config)
             throw new Error("Database or config not initialized");
         for (const database of databases) {
             if (database.$id === "migrations")
                 continue;
-            await this.createOrUpdateCollections(database);
+            await this.createOrUpdateCollections(database, undefined, collections);
         }
     }
-    async createOrUpdateCollections(database, deletedCollections) {
+    async createOrUpdateCollections(database, deletedCollections, collections = []) {
         await this.init();
         if (!this.database || !this.config)
             throw new Error("Database or config not initialized");
-        await createOrUpdateCollections(this.database, database.$id, this.config, deletedCollections);
+        await createOrUpdateCollections(this.database, database.$id, this.config, deletedCollections, collections);
     }
     async generateSchemas() {
         await this.init();
@@ -138,13 +161,17 @@ export class UtilsController {
             throw new Error("Config not initialized");
         await generateSchemas(this.config, this.appwriteFolderPath);
     }
-    async importData(options) {
+    async importData(options = {}) {
         await this.init();
-        if (!this.config || !this.database || !this.storage)
-            throw new Error("Config, database, or storage not initialized");
+        if (!this.database)
+            throw new Error("Database not initialized");
+        if (!this.storage)
+            throw new Error("Storage not initialized");
+        if (!this.config)
+            throw new Error("Config not initialized");
         const importDataActions = new ImportDataActions(this.database, this.storage, this.config, this.converterDefinitions, this.validityRuleDefinitions, this.afterImportActionsDefinitions);
-        const importController = new ImportController(this.config, this.database, this.storage, this.appwriteFolderPath, importDataActions, options);
-        await importController.run();
+        const importController = new ImportController(this.config, this.database, this.storage, this.appwriteFolderPath, importDataActions, options, options.databases);
+        await importController.run(options.collections);
     }
     async synchronizeConfigurations(databases, config) {
         await this.init();
@@ -156,14 +183,17 @@ export class UtilsController {
         const appwriteToX = new AppwriteToX(configToUse, this.appwriteFolderPath, this.storage);
         await appwriteToX.toSchemas(databases);
     }
-    async syncDb() {
+    async syncDb(databases = [], collections = []) {
         await this.init();
         if (!this.database)
             throw new Error("Database not initialized");
-        const databases = await fetchAllDatabases(this.database);
+        if (databases.length === 0) {
+            const allDatabases = await fetchAllDatabases(this.database);
+            databases = allDatabases;
+        }
         await this.ensureDatabasesExist(databases);
         await this.ensureDatabaseConfigBucketsExist(databases);
-        await this.createOrUpdateCollectionsForDatabases(databases);
+        await this.createOrUpdateCollectionsForDatabases(databases, collections);
     }
     getAppwriteFolderPath() {
         return this.appwriteFolderPath;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.9.77",
+  "version": "0.9.78",
   "main": "src/main.ts",
   "type": "module",
   "repository": {
@@ -31,14 +31,15 @@
   },
   "dependencies": {
     "@types/inquirer": "^9.0.7",
-    "appwrite-utils": "^0.3.8",
+    "appwrite-utils": "^0.3.9",
+    "chalk": "^5.3.0",
     "commander": "^12.1.0",
     "inquirer": "^9.3.6",
     "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",
     "luxon": "^3.5.0",
     "nanostores": "^0.10.3",
-    "node-appwrite": "14.0.0",
+    "node-appwrite": "^14.1.0",
     "tsx": "^4.17.0",
     "ulidx": "^2.4.0",
     "winston": "^3.14.2",

package/src/collections/attributes.ts

@@ -7,6 +7,7 @@ import {
 import { nameToIdMapping, enqueueOperation } from "../migrations/queue.js";
 import _ from "lodash";
 import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import chalk from "chalk";
 
 const attributesSame = (
   databaseAttribute: Attribute,
@@ -85,6 +86,7 @@ export const createOrUpdateAttribute = async (
       (attr) => attr.key === attribute.key
     ) as unknown as any;
     foundAttribute = parseAttribute(collectionAttr);
+    // console.log(`Found attribute: ${JSON.stringify(foundAttribute)}`);
   } catch (error) {
     foundAttribute = undefined;
   }
@@ -96,7 +98,10 @@ export const createOrUpdateAttribute = async (
     // console.log(
     //   `Updating attribute with same key ${attribute.key} but different values`
     // );
-    finalAttribute = attribute;
+    finalAttribute = {
+      ...foundAttribute,
+      ...attribute,
+    };
     action = "update";
   } else if (!updateEnabled && foundAttribute && !attributesSame(foundAttribute, attribute)) {
     await db.deleteAttribute(dbId, collection.$id, attribute.key);
@@ -120,9 +125,9 @@ export const createOrUpdateAttribute = async (
           relatedCollectionId!
         );
       } catch (e) {
-        console.log(
-          `Collection not found: ${finalAttribute.relatedCollection} when nameToIdMapping was set`
-        );
+        // console.log(
+        //   `Collection not found: ${finalAttribute.relatedCollection} when nameToIdMapping was set`
+        // );
         collectionFoundViaRelatedCollection = undefined;
       }
     } else {
@@ -139,7 +144,7 @@ export const createOrUpdateAttribute = async (
       }
     }
     if (!(relatedCollectionId && collectionFoundViaRelatedCollection)) {
-      console.log(`Enqueueing operation for attribute: ${finalAttribute.key}`);
+      // console.log(`Enqueueing operation for attribute: ${finalAttribute.key}`);
       enqueueOperation({
         type: "attribute",
         collectionId: collection.$id,
@@ -150,7 +155,7 @@ export const createOrUpdateAttribute = async (
       return;
     }
   }
-  finalAttribute = attributeSchema.parse(finalAttribute);
+  finalAttribute = parseAttribute(finalAttribute);
   // console.log(`Final Attribute: ${JSON.stringify(finalAttribute)}`);
   switch (finalAttribute.type) {
     case "string":
@@ -463,7 +468,7 @@ export const createUpdateCollectionAttributes = async (
   attributes: Attribute[]
 ): Promise<void> => {
   console.log(
-    `Creating/Updating attributes for collection: ${collection.name}`
+    chalk.green(`Creating/Updating attributes for collection: ${collection.name}`)
   );
 
   const batchSize = 3;

package/src/collections/methods.ts

@@ -6,7 +6,7 @@ import {
   Query,
   type Models,
 } from "node-appwrite";
-import type { AppwriteConfig, CollectionCreate } from "appwrite-utils";
+import type { AppwriteConfig, CollectionCreate, Indexes } from "appwrite-utils";
 import { nameToIdMapping, processQueue } from "../migrations/queue.js";
 import { createUpdateCollectionAttributes } from "./attributes.js";
 import { createOrUpdateIndexes } from "./indexes.js";
@@ -124,6 +124,22 @@ export const fetchAndCacheCollectionByName = async (
   }
 };
 
+async function wipeDocumentsFromCollection(database: Databases, databaseId: string, collectionId: string) {
+  const initialDocuments = await database.listDocuments(databaseId, collectionId, [Query.limit(1000)]);
+  let documents = initialDocuments.documents;
+  while (documents.length === 1000) {
+    const docsResponse = await database.listDocuments(databaseId, collectionId, [Query.limit(1000)]);
+    documents = documents.concat(docsResponse.documents);
+  }
+  const batchDeletePromises = documents.map(doc => database.deleteDocument(databaseId, collectionId, doc.$id));
+  const maxStackSize = 100;
+  for (let i = 0; i < batchDeletePromises.length; i += maxStackSize) {
+    await Promise.all(batchDeletePromises.slice(i, i + maxStackSize));
+    await delay(100);
+  }
+  console.log(`Deleted ${documents.length} documents from collection ${collectionId}`);
+}
+
 export const wipeDatabase = async (
   database: Databases,
   databaseId: string
@@ -146,6 +162,20 @@ export const wipeDatabase = async (
   return collectionsDeleted;
 };
 
+export const wipeCollection = async (
+  database: Databases,
+  databaseId: string,
+  collectionId: string
+): Promise<void> => {
+  const collections = await database.listCollections(databaseId, [Query.equal("$id", collectionId)]);
+  if (collections.total === 0) {
+    console.log(`Collection ${collectionId} not found`);
+    return;
+  }
+  const collection = collections.collections[0];
+  await wipeDocumentsFromCollection(database, databaseId, collection.$id);
+};
+
 export const generateSchemas = async (
   config: AppwriteConfig,
   appwriteFolderPath: string
@@ -158,38 +188,45 @@ export const createOrUpdateCollections = async (
   database: Databases,
   databaseId: string,
   config: AppwriteConfig,
-  deletedCollections?: { collectionId: string; collectionName: string }[]
+  deletedCollections?: { collectionId: string; collectionName: string }[],
+  selectedCollections: Models.Collection[] = []
 ): Promise<void> => {
-  const configCollections = config.collections;
-  if (!configCollections) {
+  const collectionsToProcess = selectedCollections.length > 0 ? selectedCollections : config.collections;
+  if (!collectionsToProcess) {
     return;
   }
   const usedIds = new Set();
 
-  for (const { attributes, indexes, ...collection } of configCollections) {
+  for (const collection of collectionsToProcess) {
+    const { attributes, indexes, ...collectionData } = collection;
+
     // Prepare permissions for the collection
     const permissions: string[] = [];
     if (collection.$permissions && collection.$permissions.length > 0) {
       for (const permission of collection.$permissions) {
-        switch (permission.permission) {
-          case "read":
-            permissions.push(Permission.read(permission.target));
-            break;
-          case "create":
-            permissions.push(Permission.create(permission.target));
-            break;
-          case "update":
-            permissions.push(Permission.update(permission.target));
-            break;
-          case "delete":
-            permissions.push(Permission.delete(permission.target));
-            break;
-          case "write":
-            permissions.push(Permission.write(permission.target));
-            break;
-          default:
-            console.log(`Unknown permission: ${permission.permission}`);
-            break;
+        if (typeof permission === 'string') {
+          permissions.push(permission);
+        } else {
+          switch (permission.permission) {
+            case "read":
+              permissions.push(Permission.read(permission.target));
+              break;
+            case "create":
+              permissions.push(Permission.create(permission.target));
+              break;
+            case "update":
+              permissions.push(Permission.update(permission.target));
+              break;
+            case "delete":
+              permissions.push(Permission.delete(permission.target));
+              break;
+            case "write":
+              permissions.push(Permission.write(permission.target));
+              break;
+            default:
+              console.log(`Unknown permission: ${permission.permission}`);
+              break;
+          }
         }
       }
     }
@@ -198,7 +235,7 @@ export const createOrUpdateCollections = async (
     let collectionsFound = await tryAwaitWithRetry(
       async () =>
         await database.listCollections(databaseId, [
-          Query.equal("name", collection.name),
+          Query.equal("name", collectionData.name),
         ])
     );
 
@@ -208,15 +245,15 @@ export const createOrUpdateCollections = async (
     // Determine the correct ID for the collection
     let collectionId: string;
     if (!collectionToUse) {
-      console.log(`Creating collection: ${collection.name}`);
+      console.log(`Creating collection: ${collectionData.name}`);
       let foundColl = deletedCollections?.find(
         (coll) =>
           coll.collectionName.toLowerCase().trim().replace(" ", "") ===
-          collection.name.toLowerCase().trim().replace(" ", "")
+          collectionData.name.toLowerCase().trim().replace(" ", "")
       );
 
-      if (collection.$id) {
-        collectionId = collection.$id;
+      if (collectionData.$id) {
+        collectionId = collectionData.$id;
       } else if (foundColl && !usedIds.has(foundColl.collectionId)) {
         collectionId = foundColl.collectionId;
       } else {
@@ -232,31 +269,31 @@ export const createOrUpdateCollections = async (
             await database.createCollection(
               databaseId,
               collectionId,
-              collection.name,
+              collectionData.name,
               permissions,
-              collection.documentSecurity ?? false,
-              collection.enabled ?? true
+              collectionData.documentSecurity ?? false,
+              collectionData.enabled ?? true
             )
         );
-        collection.$id = collectionToUse!.$id;
-        nameToIdMapping.set(collection.name, collectionToUse!.$id);
+        collectionData.$id = collectionToUse!.$id;
+        nameToIdMapping.set(collectionData.name, collectionToUse!.$id);
       } catch (error) {
         console.error(
-          `Failed to create collection ${collection.name} with ID ${collectionId}: ${error}`
+          `Failed to create collection ${collectionData.name} with ID ${collectionId}: ${error}`
         );
         continue;
       }
     } else {
-      console.log(`Collection ${collection.name} exists, updating it`);
+      console.log(`Collection ${collectionData.name} exists, updating it`);
       await tryAwaitWithRetry(
         async () =>
           await database.updateCollection(
             databaseId,
             collectionToUse!.$id,
-            collection.name,
+            collectionData.name,
             permissions,
-            collection.documentSecurity ?? false,
-            collection.enabled ?? true
+            collectionData.documentSecurity ?? false,
+            collectionData.enabled ?? true
           )
       );
     }
@@ -270,6 +307,7 @@ export const createOrUpdateCollections = async (
       database,
       databaseId,
       collectionToUse!,
+      // @ts-expect-error
       attributes
     );
 
@@ -281,7 +319,7 @@ export const createOrUpdateCollections = async (
       databaseId,
       database,
       collectionToUse!.$id,
-      indexes ?? []
+      (indexes ?? []) as Indexes,
     );
 
     // Add delay after creating indexes