appwrite-utils-cli 1.5.2 → 1.6.0

package/src/migrations/afterImportActions.ts

@@ -4,7 +4,6 @@ import {
   Query,
   ID,
   type Models,
-  Client,
   Compression,
 } from "node-appwrite";
 import { InputFile } from "node-appwrite/file";
@@ -17,24 +16,16 @@ import {
   type AfterImportActions,
   type AppwriteConfig,
 } from "appwrite-utils";
+import { getClientFromConfig } from "../utils/getClientFromConfig.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
 
 export const getDatabaseFromConfig = (config: AppwriteConfig) => {
-  if (!config.appwriteClient) {
-    config.appwriteClient = new Client()
-      .setEndpoint(config.appwriteEndpoint)
-      .setProject(config.appwriteProject)
-      .setKey(config.appwriteKey);
-  }
+  getClientFromConfig(config); // Sets config.appwriteClient if missing
   return new Databases(config.appwriteClient!);
 };
 
 export const getStorageFromConfig = (config: AppwriteConfig) => {
-  if (!config.appwriteClient) {
-    config.appwriteClient = new Client()
-      .setEndpoint(config.appwriteEndpoint)
-      .setProject(config.appwriteProject)
-      .setKey(config.appwriteKey);
-  }
+  getClientFromConfig(config); // Sets config.appwriteClient if missing
   return new Storage(config.appwriteClient!);
 };
 
@@ -52,7 +43,11 @@ export const afterImportActions = {
         async () => await db.updateDocument(dbId, collId, docId, data)
       );
     } catch (error) {
-      console.error("Error updating document: ", error);
+      MessageFormatter.error(
+        "Error updating document",
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Import" }
+      );
     }
   },
   checkAndUpdateFieldInDocument: async (
@@ -78,7 +73,11 @@ export const afterImportActions = {
         );
       }
     } catch (error) {
-      console.error("Error updating document: ", error);
+      MessageFormatter.error(
+        "Error updating document",
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Import" }
+      );
     }
   },
   setFieldFromOtherCollectionDocument: async (
@@ -133,13 +132,15 @@ export const afterImportActions = {
         );
       }
 
-      console.log(
-        `Field ${fieldName} updated successfully in document ${docId}.`
+      MessageFormatter.success(
+        `Field ${fieldName} updated successfully in document ${docId}`,
+        { prefix: "Import" }
       );
     } catch (error) {
-      console.error(
-        "Error setting field from other collection document: ",
-        error
+      MessageFormatter.error(
+        "Error setting field from other collection document",
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Import" }
       );
     }
   },
@@ -239,14 +240,16 @@ export const afterImportActions = {
             )
         );
 
-        console.log(
-          `Field ${fieldName} updated successfully in document ${docId} with ${documentIds.length} document IDs.`
+        MessageFormatter.success(
+          `Field ${fieldName} updated successfully in document ${docId} with ${documentIds.length} document IDs`,
+          { prefix: "Import" }
         );
       }
     } catch (error) {
-      console.error(
-        "Error setting field from other collection documents: ",
-        error
+      MessageFormatter.error(
+        "Error setting field from other collection documents",
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Import" }
       );
     }
   },
@@ -339,14 +342,16 @@ export const afterImportActions = {
             )
         );
 
-        console.log(
-          `Field ${fieldName} updated successfully in document ${docId} with values from field ${targetField}.`
+        MessageFormatter.success(
+          `Field ${fieldName} updated successfully in document ${docId} with values from field ${targetField}`,
+          { prefix: "Import" }
         );
       }
     } catch (error) {
-      console.error(
-        "Error setting field from other collection documents: ",
-        error
+      MessageFormatter.error(
+        "Error setting field from other collection documents",
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Import" }
       );
     }
   },
@@ -410,7 +415,11 @@ export const afterImportActions = {
         );
       }
     } catch (error) {
-      console.error("Error creating or getting bucket: ", error);
+      MessageFormatter.error(
+        "Error creating or getting bucket",
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Import" }
+      );
     }
   },
   createFileAndUpdateField: async (
@@ -435,16 +444,19 @@ export const afterImportActions = {
       //   `Processing field ${fieldName} in collection ${collId} for document ${docId} in database ${dbId} in bucket ${bucketId} with path ${filePath} and name ${fileName}...`
       // );
       if (filePath.length === 0 || fileName.length === 0) {
-        console.error(
-          `File path or name is empty for field ${fieldName} in collection ${collId}, skipping...`
+        MessageFormatter.error(
+          `File path or name is empty for field ${fieldName} in collection ${collId}, skipping...`,
+          undefined,
+          { prefix: "Import" }
         );
         return;
       }
 
       let isArray = false;
       if (!attribute) {
-        console.log(
-          `Field ${fieldName} not found in collection ${collId}, weird, skipping...`
+        MessageFormatter.warning(
+          `Field ${fieldName} not found in collection ${collId}, weird, skipping...`,
+          { prefix: "Import" }
         );
         return;
       } else if (attribute.array === true) {
@@ -478,8 +490,10 @@ export const afterImportActions = {
           async () => await fetch(filePath)
         );
         if (!response.ok)
-          console.error(
-            `Failed to fetch ${filePath}: ${response.statusText} for document ${docId} with field ${fieldName}`
+          MessageFormatter.error(
+            `Failed to fetch ${filePath}: ${response.statusText} for document ${docId} with field ${fieldName}`,
+            undefined,
+            { prefix: "Import" }
           );
 
         // Use arrayBuffer if buffer is not available
@@ -495,7 +509,10 @@ export const afterImportActions = {
           async () => await storage.createFile(bucketId, ID.unique(), inputFile)
         );
 
-        console.log("Created file from URL: ", file.$id);
+        MessageFormatter.success(
+          `Created file from URL: ${file.$id}`,
+          { prefix: "Import" }
+        );
 
         // After uploading, adjust the updateData based on whether the field is an array or not
         if (isArray) {
@@ -516,8 +533,10 @@ export const afterImportActions = {
         const files = fs.readdirSync(filePath);
         const fileFullName = files.find((file) => file.includes(fileName));
         if (!fileFullName) {
-          console.error(
-            `File starting with '${fileName}' not found in '${filePath}'`
+          MessageFormatter.error(
+            `File starting with '${fileName}' not found in '${filePath}'`,
+            undefined,
+            { prefix: "Import" }
           );
           return;
         }
@@ -538,15 +557,23 @@ export const afterImportActions = {
               [fieldName]: updateData,
             })
         );
-        console.log("Created file from path: ", file.$id);
+        MessageFormatter.success(
+          `Created file from path: ${file.$id}`,
+          { prefix: "Import" }
+        );
       }
     } catch (error) {
       logger.error(
         `Error creating file and updating field, params were:\ndbId: ${dbId}, collId: ${collId}, docId: ${docId}, fieldName: ${fieldName}, filePath: ${filePath}, fileName: ${fileName}\n\nError: ${error}`
       );
-      console.error("Error creating file and updating field: ", error);
-      console.log(
-        `Params were: dbId: ${dbId}, collId: ${collId}, docId: ${docId}, fieldName: ${fieldName}, filePath: ${filePath}, fileName: ${fileName}`
+      MessageFormatter.error(
+        "Error creating file and updating field",
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Import" }
+      );
+      MessageFormatter.info(
+        `Params were: dbId: ${dbId}, collId: ${collId}, docId: ${docId}, fieldName: ${fieldName}, filePath: ${filePath}, fileName: ${fileName}`,
+        { prefix: "Import" }
       );
     }
   },

package/src/migrations/appwriteToX.ts

@@ -28,6 +28,7 @@ import {
 import { getDatabaseFromConfig } from "./afterImportActions.js";
 import { listBuckets } from "../storage/methods.js";
 import { listFunctions, listFunctionDeployments } from "../functions/methods.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
 
 export class AppwriteToX {
   config: AppwriteConfig;
@@ -96,19 +97,62 @@ export class AppwriteToX {
   async appwriteSync(config: AppwriteConfig, databases?: Models.Database[]) {
     const db = getDatabaseFromConfig(config);
     if (!databases) {
-      databases = await fetchAllDatabases(db);
+      try {
+        MessageFormatter.info("Fetching remote databases...", { prefix: "Migration" });
+        databases = await fetchAllDatabases(db);
+        MessageFormatter.info(`Found ${databases.length} remote databases`, { prefix: "Migration" });
+      } catch (error) {
+        MessageFormatter.error(
+          "Failed to fetch remote databases",
+          error instanceof Error ? error : new Error(String(error)),
+          { prefix: "Migration" }
+        );
+        throw new Error(`Database fetch failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      }
     }
     let updatedConfig: AppwriteConfig = { ...config };
 
+    // Initialize databases array if it doesn't exist
+    if (!updatedConfig.databases) {
+      updatedConfig.databases = [];
+    }
+
+    // Sync remote databases to local config - add missing ones
+    MessageFormatter.info(`Syncing ${databases.length} remote databases with local config...`, { prefix: "Migration" });
+    let addedCount = 0;
+    let updatedCount = 0;
+
+    for (const remoteDb of databases) {
+      // Check if this database already exists in the config
+      const existingDbIndex = updatedConfig.databases.findIndex(
+        (localDb) => localDb.$id === remoteDb.$id
+      );
+
+      if (existingDbIndex === -1) {
+        // Database doesn't exist locally, add it
+        MessageFormatter.success(`Adding new database to config: ${remoteDb.name} (${remoteDb.$id})`, { prefix: "Migration" });
+        updatedConfig.databases.push({
+          $id: remoteDb.$id,
+          name: remoteDb.name,
+        });
+        addedCount++;
+      } else {
+        // Database exists, update name if different
+        if (updatedConfig.databases[existingDbIndex].name !== remoteDb.name) {
+          MessageFormatter.info(`Updating database name: ${updatedConfig.databases[existingDbIndex].name} -> ${remoteDb.name}`, { prefix: "Migration" });
+          updatedConfig.databases[existingDbIndex].name = remoteDb.name;
+          updatedCount++;
+        }
+      }
+    }
+
+    MessageFormatter.success(`Database sync summary: ${addedCount} added, ${updatedCount} updated, ${updatedConfig.databases.length} total`, { prefix: "Migration" });
+
     // Fetch all buckets
     const allBuckets = await listBuckets(this.storage);
 
     // Loop through each database
     for (const database of databases) {
-      if (!this.config.useMigrations && database.name.toLowerCase() === "migrations") {
-        continue;
-      }
-
       // Match bucket to database
       const matchedBucket = allBuckets.buckets.find((bucket) =>
         bucket.$id.toLowerCase().includes(database.$id.toLowerCase())
@@ -139,7 +183,7 @@ export class AppwriteToX {
         updatedConfig.collections = [];
       }
       for (const collection of collections) {
-        console.log(`Processing collection: ${collection.name}`);
+        MessageFormatter.processing(`Processing collection: ${collection.name}`, { prefix: "Migration" });
         const existingCollectionIndex = updatedConfig.collections.findIndex(
           (c) => c.name === collection.name
         );
@@ -161,23 +205,30 @@ export class AppwriteToX {
             attribute.type === "relationship" &&
             attribute.relatedCollection
           ) {
-            console.log(
-              `Fetching related collection for ID: ${attribute.relatedCollection}`
+            MessageFormatter.info(
+              `Fetching related collection for ID: ${attribute.relatedCollection}`,
+              { prefix: "Migration" }
             );
             try {
               const relatedCollectionPulled = await db.getCollection(
                 database.$id,
                 attribute.relatedCollection
               );
-              console.log(
-                `Fetched Collection Name: ${relatedCollectionPulled.name}`
+              MessageFormatter.info(
+                `Fetched Collection Name: ${relatedCollectionPulled.name}`,
+                { prefix: "Migration" }
               );
               attribute.relatedCollection = relatedCollectionPulled.name;
-              console.log(
-                `Updated attribute.relatedCollection to: ${attribute.relatedCollection}`
+              MessageFormatter.info(
+                `Updated attribute.relatedCollection to: ${attribute.relatedCollection}`,
+                { prefix: "Migration" }
               );
             } catch (error) {
-              console.log("Error fetching related collection:", error);
+              MessageFormatter.error(
+                "Error fetching related collection",
+                error instanceof Error ? error : new Error(String(error)),
+                { prefix: "Migration" }
+              );
             }
           }
         }
@@ -215,8 +266,9 @@ export class AppwriteToX {
         }
       }
 
-      console.log(
-        `Processed ${collections.length} collections in ${database.name}`
+      MessageFormatter.success(
+        `Processed ${collections.length} collections in ${database.name}`,
+        { prefix: "Migration" }
       );
     }
     // Add unmatched buckets as global buckets
@@ -260,32 +312,46 @@ export class AppwriteToX {
       })
     );
 
-    // Make sure to update the config with all changes
+    // Make sure to update the config with all changes including databases
     updatedConfig.functions = this.updatedConfig.functions;
     this.updatedConfig = updatedConfig;
+    MessageFormatter.success(`Sync completed - ${updatedConfig.databases.length} databases, ${updatedConfig.collections?.length || 0} collections, ${updatedConfig.buckets?.length || 0} buckets, ${updatedConfig.functions?.length || 0} functions`, { prefix: "Migration" });
   }
 
   async toSchemas(databases?: Models.Database[]) {
-    await this.appwriteSync(this.config, databases);
-    const generator = new SchemaGenerator(
-      this.updatedConfig,
-      this.appwriteFolderPath
-    );
+    try {
+      MessageFormatter.info("Starting sync-from-Appwrite process...", { prefix: "Migration" });
+      await this.appwriteSync(this.config, databases);
 
-    // Check if this is a YAML-based project
-    const yamlConfigPath = findYamlConfig(this.appwriteFolderPath);
-    const isYamlProject = !!yamlConfigPath;
+      const generator = new SchemaGenerator(
+        this.updatedConfig,
+        this.appwriteFolderPath
+      );
+
+      // Check if this is a YAML-based project
+      const yamlConfigPath = findYamlConfig(this.appwriteFolderPath);
+      const isYamlProject = !!yamlConfigPath;
 
-    if (isYamlProject) {
-      console.log("📄 Detected YAML configuration - generating YAML collection definitions");
-      generator.updateYamlCollections();
-      await generator.updateConfig(this.updatedConfig, true);
-    } else {
-      console.log("📝 Generating TypeScript collection definitions");
-      generator.updateTsSchemas();
-      await generator.updateConfig(this.updatedConfig, false);
+      if (isYamlProject) {
+        MessageFormatter.info("Detected YAML configuration - generating YAML collection definitions", { prefix: "Migration" });
+        generator.updateYamlCollections();
+        await generator.updateConfig(this.updatedConfig, true);
+      } else {
+        MessageFormatter.info("Generating TypeScript collection definitions", { prefix: "Migration" });
+        generator.updateTsSchemas();
+        await generator.updateConfig(this.updatedConfig, false);
+      }
+
+      MessageFormatter.info("Generating Zod schemas from synced collections...", { prefix: "Migration" });
+      generator.generateSchemas();
+      MessageFormatter.success("Sync-from-Appwrite process completed successfully", { prefix: "Migration" });
+    } catch (error) {
+      MessageFormatter.error(
+        "Error during sync-from-Appwrite process",
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Migration" }
+      );
+      throw error;
     }
-    
-    generator.generateSchemas();
   }
 }

package/src/migrations/dataLoader.ts

@@ -21,9 +21,11 @@ import { ID, Users, type Databases } from "node-appwrite";
 import { logger } from "../shared/logging.js";
 import { findOrCreateOperation, updateOperation } from "../shared/migrationHelpers.js";
 import { AuthUserCreateSchema } from "../schemas/authUser.js";
+import { LegacyAdapter } from "../adapters/LegacyAdapter.js";
 import { UsersController } from "../users/methods.js";
 import { finalizeByAttributeMap } from "../utils/helperFunctions.js";
 import { isEmpty } from "es-toolkit/compat";
+import { MessageFormatter } from "../shared/messageFormatter.js";
 
 // Define a schema for the structure of collection import data using Zod for validation
 export const CollectionImportDataSchema = z.object({
@@ -185,9 +187,9 @@ export class DataLoader {
   loadData(importDef: ImportDef): any[] {
     // Simply join appwriteFolderPath with the importDef.filePath
     const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
-    console.log(`Loading data from: ${filePath}`); // Add logging
+    MessageFormatter.info(`Loading data from: ${filePath}`, { prefix: "Data" });
     if (!fs.existsSync(filePath)) {
-      console.error(`File not found: ${filePath}`);
+      MessageFormatter.error(`File not found: ${filePath}`, undefined, { prefix: "Data" });
       return [];
     }
 
@@ -197,7 +199,7 @@ export class DataLoader {
       ? JSON.parse(rawData)[importDef.basePath]
       : JSON.parse(rawData);
 
-    console.log(`Loaded ${parsedData?.length || 0} items from ${filePath}`);
+    MessageFormatter.success(`Loaded ${parsedData?.length || 0} items from ${filePath}`, { prefix: "Data" });
     return parsedData;
   }
 
@@ -313,10 +315,12 @@ export class DataLoader {
         collection.$id = collectionExists.$id;
         this.config.collections[index] = collectionConfig;
         // Find or create an import operation for the collection
+        const adapter = new LegacyAdapter(this.database);
         const collectionImportOperation = await findOrCreateOperation(
-          this.database,
-          collection.$id!,
-          "importData"
+          adapter,
+          dbId,
+          "importData",
+          collection.$id!
         );
         // Store the operation ID in the map
         this.collectionImportOperations.set(
@@ -373,13 +377,14 @@ export class DataLoader {
 
   // Main method to start the data loading process for a given database ID
   async start(dbId: string) {
-    console.log("---------------------------------");
-    console.log(`Starting data setup for database: ${dbId}`);
-    console.log("---------------------------------");
+    MessageFormatter.divider();
+    MessageFormatter.info(`Starting data setup for database: ${dbId}`, { prefix: "Data" });
+    MessageFormatter.divider();
     await this.setupMaps(dbId);
     const allUsers = await this.getAllUsers();
-    console.log(
-      `Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`
+    MessageFormatter.info(
+      `Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`,
+      { prefix: "Data" }
     );
     await new Promise((resolve) => setTimeout(resolve, 5000));
     // Iterate over the configured databases to find the matching one
@@ -431,17 +436,17 @@ export class DataLoader {
           this.prepareUpdateData(db, collection, updateDef);
         }
       }
-      console.log("Running update references");
+      MessageFormatter.info("Running update references", { prefix: "Data" });
       // this.dealWithMergedUsers();
       this.updateOldReferencesForNew();
-      console.log("Done running update references");
+      MessageFormatter.success("Done running update references", { prefix: "Data" });
     }
     // for (const collection of this.config.collections) {
     //   this.resolveDataItemRelationships(collection);
     // }
-    console.log("---------------------------------");
-    console.log(`Data setup for database: ${dbId} completed`);
-    console.log("---------------------------------");
+    MessageFormatter.divider();
+    MessageFormatter.success(`Data setup for database: ${dbId} completed`, { prefix: "Data" });
+    MessageFormatter.divider();
     if (this.shouldWriteFile) {
       this.writeMapsToJsonFile();
     }
@@ -528,8 +533,9 @@ export class DataLoader {
 
       if (!collectionData || !collectionData.data) continue;
 
-      console.log(
-        `Updating references for collection: ${collectionConfig.name}`
+      MessageFormatter.processing(
+        `Updating references for collection: ${collectionConfig.name}`,
+        { prefix: "Data" }
       );
 
       let needsUpdate = false;
@@ -728,10 +734,10 @@ export class DataLoader {
       const outputFile = path.join(outputDir, fileName);
       fs.writeFile(outputFile, JSON.stringify(data, null, 2), "utf8", (err) => {
         if (err) {
-          console.error(`Error writing data to ${fileName}:`, err);
+          MessageFormatter.error(`Error writing data to ${fileName}`, err instanceof Error ? err : new Error(String(err)), { prefix: "Data" });
           return;
         }
-        console.log(`Data successfully written to ${fileName}`);
+        MessageFormatter.success(`Data successfully written to ${fileName}`, { prefix: "Data" });
       });
     };
 
@@ -951,11 +957,13 @@ export class DataLoader {
       this.oldIdToNewIdPerCollectionMap
         .set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
         .get(this.getCollectionKey(collection.name));
+    const adapter = new LegacyAdapter(this.database);
     if (!operationId) {
       const collectionImportOperation = await findOrCreateOperation(
-        this.database,
-        collection.$id!,
-        "importData"
+        adapter,
+        db.$id,
+        "importData",
+        collection.$id!
       );
       // Store the operation ID in the map
       this.collectionImportOperations.set(
@@ -964,10 +972,12 @@ export class DataLoader {
       );
       operationId = collectionImportOperation.$id;
     }
-    await updateOperation(this.database, operationId, {
-      status: "ready",
-      total: rawData.length,
-    });
+    if (operationId) {
+      await updateOperation(adapter, db.$id, operationId, {
+        status: "ready",
+        total: rawData.length,
+      });
+    }
     // Retrieve the current user data and the current collection data from the import map
     const currentUserData = this.importMap.get(this.getCollectionKey("users"));
     const currentData = this.importMap.get(
@@ -1178,11 +1188,13 @@ export class DataLoader {
     let operationId = this.collectionImportOperations.get(
       this.getCollectionKey(collection.name)
     );
+    const adapter = new LegacyAdapter(this.database);
     if (!operationId) {
       const collectionImportOperation = await findOrCreateOperation(
-        this.database,
-        collection.$id!,
-        "importData"
+        adapter,
+        db.$id,
+        "importData",
+        collection.$id!
       );
       // Store the operation ID in the map
       this.collectionImportOperations.set(
@@ -1191,10 +1203,12 @@ export class DataLoader {
       );
       operationId = collectionImportOperation.$id;
     }
-    await updateOperation(this.database, operationId, {
-      status: "ready",
-      total: rawData.length,
-    });
+    if (operationId) {
+      await updateOperation(adapter, db.$id, operationId, {
+        status: "ready",
+        total: rawData.length,
+      });
+    }
     // Initialize a new map for old ID to new ID mappings
     const oldIdToNewIdMapNew = new Map<string, string>();
     // Retrieve or initialize the collection-specific old ID to new ID map