appwrite-utils-cli 0.9.77 → 0.9.79

package/src/migrations/importController.ts

@@ -67,7 +67,7 @@ export class ImportController {
     this.databasesToRun = databasesToRun || [];
   }
 
-  async run() {
+  async run(specificCollections?: string[]) {
     let databasesToProcess: Models.Database[];
 
     if (this.databasesToRun.length > 0) {
@@ -101,9 +101,9 @@ export class ImportController {
           this.setupOptions.shouldWriteFile
         );
         await dataLoader.start(db.$id);
-        await this.importCollections(db, dataLoader);
+        await this.importCollections(db, dataLoader, specificCollections);
         await resolveAndUpdateRelationships(db.$id, this.database, this.config);
-        await this.executePostImportActions(db.$id, dataLoader);
+        await this.executePostImportActions(db.$id, dataLoader, specificCollections);
       } else if (databaseRan.$id !== db.$id) {
         await this.updateOthersToFinalData(databaseRan, db);
       }
@@ -142,307 +142,195 @@ export class ImportController {
     }
   }
 
-  async importCollections(db: ConfigDatabase, dataLoader: DataLoader) {
-    if (!this.config.collections) {
-      return;
-    }
-    for (const collection of this.config.collections) {
-      let isUsersCollection =
-        dataLoader.getCollectionKey(this.config.usersCollectionName) ===
-        dataLoader.getCollectionKey(collection.name);
-      const importOperationId = dataLoader.collectionImportOperations.get(
-        dataLoader.getCollectionKey(collection.name)
-      );
-      const createBatches = (finalData: CollectionImportData["data"]) => {
-        let maxBatchLength = 100;
-        const finalBatches: CollectionImportData["data"][] = [];
-        for (let i = 0; i < finalData.length; i++) {
-          if (i % maxBatchLength === 0) {
-            finalBatches.push([]);
-          }
-          finalBatches[finalBatches.length - 1].push(finalData[i]);
-        }
-        return finalBatches;
-      };
+  async importCollections(db: ConfigDatabase, dataLoader: DataLoader, specificCollections?: string[]) {
+    const collectionsToImport = specificCollections || (this.config.collections ? this.config.collections.map(c => c.name) : []);
 
-      if (isUsersCollection && !this.hasImportedUsers) {
-        const usersDataMap = dataLoader.importMap.get(
-          dataLoader.getCollectionKey("users")
+    for (const collection of this.config.collections || []) {
+      if (collectionsToImport.includes(collection.name)) {
+        let isUsersCollection =
+          dataLoader.getCollectionKey(this.config.usersCollectionName) ===
+          dataLoader.getCollectionKey(collection.name);
+        const importOperationId = dataLoader.collectionImportOperations.get(
+          dataLoader.getCollectionKey(collection.name)
         );
-        const usersData = usersDataMap?.data;
-        const usersController = new UsersController(this.config, this.database);
-        if (usersData) {
-          console.log("Found users data", usersData.length);
-          const userDataBatches = createBatches(usersData);
-          for (const batch of userDataBatches) {
-            console.log("Importing users batch", batch.length);
-            const userBatchPromises = batch
-              .filter((item) => {
-                let itemId: string | undefined;
-                if (item.finalData.userId) {
-                  itemId = item.finalData.userId;
-                } else if (item.finalData.docId) {
-                  itemId = item.finalData.docId;
-                }
-                if (!itemId) {
-                  return false;
+        const createBatches = (finalData: CollectionImportData["data"]) => {
+          let maxBatchLength = 100;
+          const finalBatches: CollectionImportData["data"][] = [];
+          for (let i = 0; i < finalData.length; i++) {
+            if (i % maxBatchLength === 0) {
+              finalBatches.push([]);
+            }
+            finalBatches[finalBatches.length - 1].push(finalData[i]);
+          }
+          return finalBatches;
+        };
+
+        if (isUsersCollection && !this.hasImportedUsers) {
+          const usersDataMap = dataLoader.importMap.get(
+            dataLoader.getCollectionKey("users")
+          );
+          const usersData = usersDataMap?.data;
+          const usersController = new UsersController(this.config, this.database);
+          if (usersData) {
+            console.log("Found users data", usersData.length);
+            const userDataBatches = createBatches(usersData);
+            for (const batch of userDataBatches) {
+              console.log("Importing users batch", batch.length);
+              const userBatchPromises = batch
+                .filter((item) => {
+                  let itemId: string | undefined;
+                  if (item.finalData.userId) {
+                    itemId = item.finalData.userId;
+                  } else if (item.finalData.docId) {
+                    itemId = item.finalData.docId;
+                  }
+                  if (!itemId) {
+                    return false;
+                  }
+                  return (
+                    item &&
+                    item.finalData &&
+                    !dataLoader.userExistsMap.has(itemId)
+                  );
+                })
+                .map((item) => {
+                  dataLoader.userExistsMap.set(
+                    item.finalData.userId ||
+                      item.finalData.docId ||
+                      item.context.userId ||
+                      item.context.docId,
+                    true
+                  );
+                  return usersController.createUserAndReturn(item.finalData);
+                });
+              const promiseResults = await Promise.allSettled(userBatchPromises);
+              for (const item of batch) {
+                if (item && item.finalData) {
+                  dataLoader.userExistsMap.set(
+                    item.finalData.userId ||
+                      item.finalData.docId ||
+                      item.context.userId ||
+                      item.context.docId,
+                    true
+                  );
                 }
-                return (
-                  item &&
-                  item.finalData &&
-                  !dataLoader.userExistsMap.has(itemId)
-                );
-              })
-              .map((item) => {
-                dataLoader.userExistsMap.set(
-                  item.finalData.userId ||
-                    item.finalData.docId ||
-                    item.context.userId ||
-                    item.context.docId,
-                  true
-                );
-                return usersController.createUserAndReturn(item.finalData);
-              });
-            const promiseResults = await Promise.allSettled(userBatchPromises);
-            for (const item of batch) {
-              if (item && item.finalData) {
-                dataLoader.userExistsMap.set(
-                  item.finalData.userId ||
-                    item.finalData.docId ||
-                    item.context.userId ||
-                    item.context.docId,
-                  true
-                );
               }
+              console.log("Finished importing users batch");
             }
-            console.log("Finished importing users batch");
+            this.hasImportedUsers = true;
+            console.log("Finished importing users");
           }
-          this.hasImportedUsers = true;
-          console.log("Finished importing users");
         }
-      }
-
-      if (!importOperationId) {
-        // Skip further processing if no import operation is found
-        continue;
-      }
-
-      const importOperation = await this.database.getDocument(
-        "migrations",
-        "currentOperations",
-        importOperationId
-      );
-      await updateOperation(this.database, importOperation.$id, {
-        status: "in_progress",
-      });
-      const collectionData = dataLoader.importMap.get(
-        dataLoader.getCollectionKey(collection.name)
-      );
-      console.log(`Processing collection: ${collection.name}...`);
-      if (!collectionData) {
-        console.log("No collection data for ", collection.name);
-        continue;
-      }
-
-      const dataSplit = createBatches(collectionData.data);
-      let processedItems = 0;
-      for (let i = 0; i < dataSplit.length; i++) {
-        const batches = dataSplit[i];
-        console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
 
-        // const documentExistsPromises = batches.map(async (item) => {
-        //   try {
-        //     const id =
-        //       item.finalData.docId ||
-        //       item.finalData.userId ||
-        //       item.context.docId ||
-        //       item.context.userId;
+        if (!importOperationId) {
+          // Skip further processing if no import operation is found
+          continue;
+        }
 
-        //     if (!item.finalData) {
-        //       return Promise.resolve(null);
-        //     }
-        //     return tryAwaitWithRetry(
-        //       async () =>
-        //         await documentExists(
-        //           this.database,
-        //           db.$id,
-        //           collection.$id,
-        //           item.finalData
-        //         )
-        //     );
-        //   } catch (error) {
-        //     console.error(error);
-        //     return Promise.resolve(null);
-        //   }
-        // });
+        const importOperation = await this.database.getDocument(
+          "migrations",
+          "currentOperations",
+          importOperationId
+        );
+        await updateOperation(this.database, importOperation.$id, {
+          status: "in_progress",
+        });
+        const collectionData = dataLoader.importMap.get(
+          dataLoader.getCollectionKey(collection.name)
+        );
+        console.log(`Processing collection: ${collection.name}...`);
+        if (!collectionData) {
+          console.log("No collection data for ", collection.name);
+          continue;
+        }
 
-        // const documentExistsResults = await Promise.all(documentExistsPromises);
+        const dataSplit = createBatches(collectionData.data);
+        let processedItems = 0;
+        for (let i = 0; i < dataSplit.length; i++) {
+          const batches = dataSplit[i];
+          console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
 
-        const batchPromises = batches.map((item, index) => {
-          try {
-            const id =
-              item.finalData.docId ||
-              item.finalData.userId ||
-              item.context.docId ||
-              item.context.userId;
+          const batchPromises = batches.map((item, index) => {
+            try {
+              const id =
+                item.finalData.docId ||
+                item.finalData.userId ||
+                item.context.docId ||
+                item.context.userId;
 
-            if (item.finalData.hasOwnProperty("userId")) {
-              delete item.finalData.userId;
-            }
-            if (item.finalData.hasOwnProperty("docId")) {
-              delete item.finalData.docId;
-            }
-            if (!item.finalData) {
+              if (item.finalData.hasOwnProperty("userId")) {
+                delete item.finalData.userId;
+              }
+              if (item.finalData.hasOwnProperty("docId")) {
+                delete item.finalData.docId;
+              }
+              if (!item.finalData) {
+                return Promise.resolve();
+              }
+              return tryAwaitWithRetry(
+                async () =>
+                  await this.database.createDocument(
+                    db.$id,
+                    collection.$id,
+                    id,
+                    item.finalData
+                  )
+              );
+            } catch (error) {
+              console.error(error);
               return Promise.resolve();
             }
-            return tryAwaitWithRetry(
-              async () =>
-                await this.database.createDocument(
-                  db.$id,
-                  collection.$id,
-                  id,
-                  item.finalData
-                )
-            );
-          } catch (error) {
-            console.error(error);
-            return Promise.resolve();
-          }
-        });
+          });
 
-        // Wait for all promises in the current batch to resolve
-        await Promise.all(batchPromises);
-        console.log(`Completed batch ${i + 1} of ${dataSplit.length}`);
+          // Wait for all promises in the current batch to resolve
+          await Promise.all(batchPromises);
+          console.log(`Completed batch ${i + 1} of ${dataSplit.length}`);
+          await updateOperation(this.database, importOperation.$id, {
+            progress: processedItems,
+          });
+        }
+        // After all batches are processed, update the operation status to completed
         await updateOperation(this.database, importOperation.$id, {
-          progress: processedItems,
+          status: "completed",
         });
       }
-      // After all batches are processed, update the operation status to completed
-      await updateOperation(this.database, importOperation.$id, {
-        status: "completed",
-      });
     }
   }
 
-  async executePostImportActions(dbId: string, dataLoader: DataLoader) {
+  async executePostImportActions(dbId: string, dataLoader: DataLoader, specificCollections?: string[]) {
+    const collectionsToProcess = specificCollections || Array.from(dataLoader.importMap.keys());
+
     // Iterate over each collection in the importMap
-    for (const [
-      collectionKey,
-      collectionData,
-    ] of dataLoader.importMap.entries()) {
-      console.log(
-        `Processing post-import actions for collection: ${collectionKey}`
-      );
+    for (const [collectionKey, collectionData] of dataLoader.importMap.entries()) {
+      if (collectionsToProcess.includes(collectionKey)) {
+        console.log(
+          `Processing post-import actions for collection: ${collectionKey}`
+        );
 
-      // Iterate over each item in the collectionData.data
-      for (const item of collectionData.data) {
-        // Assuming each item has attributeMappings that contain actions to be executed
-        if (item.importDef && item.importDef.attributeMappings) {
-          // Use item.context as the context for action execution
-          const context = item.context; // Directly use item.context as the context for action execution
-          // Iterate through attributeMappings to execute actions
-          try {
-            // Execute post-import actions for the current attributeMapping
-            // Pass item.finalData as the data to be processed along with the context
-            await this.importDataActions.executeAfterImportActions(
-              item.finalData,
-              item.importDef.attributeMappings,
-              context
-            );
-          } catch (error) {
-            console.error(
-              `Failed to execute post-import actions for item in collection ${collectionKey}:`,
-              error
-            );
-            // Handle error (e.g., log, retry, continue with next action)
+        // Iterate over each item in the collectionData.data
+        for (const item of collectionData.data) {
+          // Assuming each item has attributeMappings that contain actions to be executed
+          if (item.importDef && item.importDef.attributeMappings) {
+            // Use item.context as the context for action execution
+            const context = item.context; // Directly use item.context as the context for action execution
+            // Iterate through attributeMappings to execute actions
+            try {
+              // Execute post-import actions for the current attributeMapping
+              // Pass item.finalData as the data to be processed along with the context
+              await this.importDataActions.executeAfterImportActions(
+                item.finalData,
+                item.importDef.attributeMappings,
+                context
+              );
+            } catch (error) {
+              console.error(
+                `Failed to execute post-import actions for item in collection ${collectionKey}:`,
+                error
+              );
+            }
           }
         }
       }
     }
   }
-
-  // async executeActionsInParallel(dbId: string, collection: ConfigCollection) {
-  //   const collectionExists = await checkForCollection(
-  //     this.database,
-  //     dbId,
-  //     collection
-  //   );
-  //   if (!collectionExists) {
-  //     logger.error(`No collection found for ${collection.name}`);
-  //     return; // Skip this iteration
-  //   }
-  //   const operations = await getAfterImportOperations(
-  //     this.database,
-  //     collectionExists.$id
-  //   );
-
-  //   for (const operation of operations) {
-  //     if (!operation.batches) {
-  //       continue;
-  //     }
-  //     const batches = operation.batches;
-  //     const promises = [];
-  //     for (const batch of batches) {
-  //       const batchId = batch;
-  //       promises.push(
-  //         this.database.getDocument("migrations", "batches", batchId)
-  //       );
-  //     }
-  //     const results = await Promise.allSettled(promises);
-  //     results.forEach((result) => {
-  //       if (result.status === "rejected") {
-  //         logger.error("A process batch promise was rejected:", result.reason);
-  //       }
-  //     });
-  //     const resultsData = results
-  //       .map((result) => (result.status === "fulfilled" ? result.value : null))
-  //       .filter((result: any) => result !== null && !result.processed)
-  //       .map((result) => BatchSchema.parse(result));
-  //     for (const batch of resultsData) {
-  //       const actionOperation = ContextObject.parse(JSON.parse(batch.data));
-  //       const { context, finalItem, attributeMappings } = actionOperation;
-  //       if (finalItem.$id && !context.docId) {
-  //         context.docId =
-  //           finalItem.$id || context.createdDoc.$id || context.$id || undefined;
-  //         logger.info(
-  //           `Setting docId to ${
-  //             finalItem.$id
-  //           } because docId not found in context, batch ${
-  //             batch.$id
-  //           }, context is ${JSON.stringify(context)}`
-  //         );
-  //       }
-  //       try {
-  //         await this.importDataActions.executeAfterImportActions(
-  //           finalItem,
-  //           attributeMappings,
-  //           context
-  //         );
-  //         // Mark batch as processed
-  //         await this.database.deleteDocument(
-  //           "migrations",
-  //           "batches",
-  //           batch.$id
-  //         );
-  //       } catch (error) {
-  //         logger.error(
-  //           `Failed to execute batch ${batch.$id}:`,
-  //           error,
-  //           "Context is :",
-  //           context
-  //         );
-  //         await this.database.deleteDocument(
-  //           "migrations",
-  //           "batches",
-  //           batch.$id
-  //         );
-  //       }
-  //     }
-
-  //     // After processing all batches, update the operation status
-  //     await updateOperation(this.database, operation.$id, {
-  //       status: "completed", // Or determine based on batch success/failure
-  //     });
-  //   }
-  // }
-}
+}

package/src/migrations/setupDatabase.ts

@@ -14,7 +14,10 @@ export const setupMigrationDatabase = async (config: AppwriteConfig) => {
   console.log("---------------------------------");
   console.log("Starting Migrations Setup");
   console.log("---------------------------------");
-  const database = new Databases(config.appwriteClient!);
+  const database = new Databases(config.appwriteClient);
+  if (!config.appwriteClient) {
+    throw new Error("Appwrite client is not initialized in the config");
+  }
   let db: Models.Database | undefined;
   const migrationCollectionsSetup = getMigrationCollectionSchemas();
 
@@ -103,9 +106,17 @@ export const setupMigrationDatabase = async (config: AppwriteConfig) => {
   console.log("---------------------------------");
 };
 
-export const ensureDatabasesExist = async (config: AppwriteConfig) => {
-  const database = new Databases(config.appwriteClient!);
-  const databasesToEnsure = config.databases;
+export const ensureDatabasesExist = async (config: AppwriteConfig, databasesToEnsure?: Models.Database[]) => {
+  if (!config.appwriteClient) {
+    throw new Error("Appwrite client is not initialized in the config");
+  }
+  const database = new Databases(config.appwriteClient);
+  const databasesToCreate = databasesToEnsure || config.databases || [];
+
+  if (!databasesToCreate.length) {
+    console.log("No databases to create");
+    return;
+  }
 
   const existingDatabases = await tryAwaitWithRetry(
     async () => await database.list([Query.limit(500)])
@@ -116,10 +127,10 @@ export const ensureDatabasesExist = async (config: AppwriteConfig) => {
   );
   if (existingDatabases.databases.length !== 0 && migrationsDatabase) {
     console.log("Creating all databases except migrations");
-    databasesToEnsure.push(migrationsDatabase);
+    databasesToCreate.push(migrationsDatabase);
   }
 
-  for (const db of databasesToEnsure) {
+  for (const db of databasesToCreate) {
     if (!existingDatabases.databases.some((d) => d.name === db.name)) {
       await tryAwaitWithRetry(
         async () => await database.create(db.$id || ulid(), db.name, true)
@@ -133,7 +144,7 @@ export const wipeOtherDatabases = async (
   database: Databases,
   databasesToKeep: Models.Database[]
 ) => {
-  console.log(`Databases to keep: ${databasesToKeep.join(", ")}`);
+  console.log(`Databases to keep: ${databasesToKeep.map(db => db.name).join(", ")}`);
   const allDatabases = await tryAwaitWithRetry(
     async () => await database.list([Query.limit(500)])
   );
@@ -151,3 +162,33 @@ export const wipeOtherDatabases = async (
     }
   }
 };
+
+export const ensureCollectionsExist = async (
+  config: AppwriteConfig,
+  database: Models.Database,
+  collectionsToEnsure?: Models.Collection[]
+) => {
+  const databaseClient = new Databases(config.appwriteClient!);
+  const collectionsToCreate = collectionsToEnsure || 
+    (config.collections ? config.collections : []);
+
+  const existingCollections = await tryAwaitWithRetry(
+    async () => await databaseClient.listCollections(database.$id, [Query.limit(500)])
+  );
+
+  for (const collection of collectionsToCreate) {
+    if (!existingCollections.collections.some((c) => c.name === collection.name)) {
+      await tryAwaitWithRetry(
+        async () => await databaseClient.createCollection(
+          database.$id,
+          ulid(),
+          collection.name,
+          undefined,
+          true,
+          true
+        )
+      );
+      console.log(`${collection.name} collection created in ${database.name}`);
+    }
+  }
+};

package/src/storage/methods.ts

@@ -155,10 +155,10 @@ export const ensureDatabaseConfigBucketsExist = async (
           );
           console.log(`Bucket ${database.bucket.$id} created successfully.`);
         } catch (createError) {
-          console.error(
-            `Failed to create bucket ${database.bucket.$id}:`,
-            createError
-          );
+          // console.error(
+          //   `Failed to create bucket ${database.bucket.$id}:`,
+          //   createError
+          // );
         }
       }
     }

package/src/utils/setupFiles.ts

@@ -74,10 +74,13 @@ const baseConfig: AppwriteConfig = {
 const collectionsConfig: { name: string; content: string }[] = [
   {
     name: "ExampleCollection",
-    content: `import { CollectionCreate } from "appwrite-utils";
+    content: `import type { CollectionCreate } from "appwrite-utils";
     
 const ExampleCollection: Partial<CollectionCreate> = {
   name: 'ExampleCollection',
+  $id: '${ulid()}',
+  documentSecurity: false,
+  enabled: true,
   $permissions: [
     { permission: 'read', target: 'any' },
     { permission: 'create', target: 'users' },