appwrite-utils-cli 0.0.286 → 0.9.0

package/src/utilsController.ts

@@ -1,194 +1,325 @@
-import { Client, Databases, Storage } from "node-appwrite";
-import { startSetup } from "./migrations/setupDatabase.js";
-import path from "path";
-import fs from "fs";
-import { load } from "js-yaml";
-import { ImportDataActions } from "./migrations/importDataActions.js";
-import { readFileSync } from "./utils/helperFunctions.js";
-import { afterImportActions } from "./migrations/afterImportActions.js";
-import {
-  type AfterImportActions,
-  type ConverterFunctions,
-  converterFunctions,
-  validationRules,
-  type ValidationRules,
-  type AppwriteConfig,
-  AppwriteConfigSchema,
-} from "appwrite-utils";
-import { ImportController } from "./migrations/importController.js";
-import _ from "lodash";
-import { AppwriteToX } from "./migrations/appwriteToX.js";
-import { loadConfig as loadTsConfig } from "./utils/loadConfigs.js";
-import { findAppwriteConfig } from "./utils/loadConfigs.js";
-
-// async function loadConfig(configPath: string) {
-//   if (!fs.existsSync(configPath)) {
-//     throw new Error(`Configuration file not found at ${configPath}`);
-//   }
-//   const configModule = await load(readFileSync(configPath), {
-//     json: true,
-//   });
-//   return AppwriteConfigSchema.parse(configModule);
-// }
-
-export interface SetupOptions {
-  sync: boolean;
-  runProd: boolean;
-  runStaging: boolean;
-  runDev: boolean;
-  doBackup: boolean;
-  wipeDatabases: boolean;
-  wipeDocumentStorage: boolean;
-  wipeUsers: boolean;
-  generateSchemas: boolean;
-  generateMockData: boolean;
-  importData: boolean;
-  checkDuplicates: boolean;
-  shouldWriteFile: boolean;
-  endpoint?: string;
-  project?: string;
-  key?: string;
-}
-
-export class UtilsController {
-  private appwriteFolderPath: string;
-  private appwriteConfigPath: string;
-  private config?: AppwriteConfig;
-  private appwriteServer?: Client;
-  private database?: Databases;
-  private storage?: Storage;
-  public converterDefinitions: ConverterFunctions = converterFunctions;
-  public validityRuleDefinitions: ValidationRules = validationRules;
-  public afterImportActionsDefinitions: AfterImportActions = afterImportActions;
-
-  constructor(currentUserDir: string) {
-    const basePath = currentUserDir; // Gets the current working directory
-    const appwriteConfigFound = findAppwriteConfig(basePath);
-    if (!appwriteConfigFound) {
-      throw new Error("Failed to find appwriteConfig.ts");
-    }
-    this.appwriteConfigPath = appwriteConfigFound;
-    this.appwriteFolderPath = path.dirname(appwriteConfigFound);
-    if (!this.appwriteFolderPath) {
-      throw new Error("Failed to get appwriteFolderPath");
-    }
-  }
-
-  // async loadCustomDefinitions(): Promise<void> {
-  //   try {
-  //     const customDefinitionsPath = path.join(
-  //       this.appwriteFolderPath,
-  //       "customDefinitions.ts"
-  //     );
-  //     if (fs.existsSync(customDefinitionsPath)) {
-  //       // Dynamically import custom definitions
-  //       const customDefinitions = (await import(
-  //         customDefinitionsPath
-  //       )) as typeof import("customDefinitions");
-  //       this.converterDefinitions = {
-  //         ...this.converterDefinitions,
-  //         ...customDefinitions.converterDefinitions,
-  //       };
-  //       this.validityRuleDefinitions = {
-  //         ...this.validityRuleDefinitions,
-  //         ...customDefinitions.validityRuleDefinitions,
-  //       };
-  //       this.afterImportActionsDefinitions = {
-  //         ...this.afterImportActionsDefinitions,
-  //         ...customDefinitions.afterImportActionsDefinitions,
-  //       };
-  //     }
-  //   } catch (error) {
-  //     console.error("Failed to load custom definitions:", error);
-  //   }
-  // }
-
-  async init(setupOptions: SetupOptions) {
-    if (!this.config) {
-      console.log("Initializing appwrite client & loading config...");
-      this.config = await loadTsConfig(this.appwriteFolderPath);
-      if (!this.config) {
-        throw new Error("Failed to load config");
-      }
-      this.appwriteServer = new Client();
-      if (setupOptions.endpoint) {
-        if (!setupOptions.project || !setupOptions.key) {
-          throw new Error(
-            "Project ID and API key required when setting endpoint"
-          );
-        }
-        this.appwriteServer
-          .setEndpoint(setupOptions.endpoint)
-          .setProject(setupOptions.project)
-          .setKey(setupOptions.key);
-      } else {
-        this.appwriteServer
-          .setEndpoint(this.config.appwriteEndpoint)
-          .setProject(this.config.appwriteProject)
-          .setKey(this.config.appwriteKey);
-      }
-      this.database = new Databases(this.appwriteServer);
-      this.storage = new Storage(this.appwriteServer);
-      this.config.appwriteClient = this.appwriteServer;
-      // await this.loadCustomDefinitions();
-    }
-  }
-
-  async run(options: SetupOptions) {
-    await this.init(options); // Ensure initialization is done
-    if (!this.database || !this.storage || !this.config) {
-      throw new Error("Database or storage not initialized");
-    }
-
-    if (options.sync) {
-      console.log("Starting synchronization with server...");
-      const appwriteToX = new AppwriteToX(this.config, this.appwriteFolderPath);
-      await appwriteToX.toSchemas();
-      console.log("Synchronization complete, YAML and Schemas updated");
-    }
-
-    // Start the setup
-    console.log(
-      "Starting setup, this step sets up migrations, runs backup, wipes databases, and updates schemas (depending on your options)..."
-    );
-    await startSetup(
-      this.database,
-      this.storage,
-      this.config,
-      options,
-      this.appwriteFolderPath
-    );
-    console.log("Setup complete.");
-
-    if (options.generateMockData) {
-      // TODO: Figure out how to do this dynamically
-      // await this.generateMockData();
-    }
-
-    if (options.importData) {
-      console.log("Starting import data...");
-      const importDataActions = new ImportDataActions(
-        this.database,
-        this.storage,
-        this.config,
-        this.converterDefinitions,
-        this.validityRuleDefinitions,
-        this.afterImportActionsDefinitions
-      );
-      const importController = new ImportController(
-        this.config!,
-        this.database!,
-        this.storage!,
-        this.appwriteFolderPath,
-        importDataActions,
-        options
-      );
-      await importController.run();
-      console.log("Import data complete.");
-    }
-
-    // if (options.checkDuplicates) {
-    //   await this.checkDuplicates();
-    // }
-  }
-}
+import { Client, Databases, Query, Storage, type Models } from "node-appwrite";
+import { type AppwriteConfig } from "appwrite-utils";
+import { loadConfig, findAppwriteConfig } from "./utils/loadConfigs.js";
+import { UsersController } from "./migrations/users.js";
+import { AppwriteToX } from "./migrations/appwriteToX.js";
+import { ImportController } from "./migrations/importController.js";
+import { ImportDataActions } from "./migrations/importDataActions.js";
+import {
+  setupMigrationDatabase,
+  ensureDatabasesExist,
+  wipeOtherDatabases,
+} from "./migrations/setupDatabase.js";
+import {
+  createOrUpdateCollections,
+  wipeDatabase,
+  generateSchemas,
+  fetchAllCollections,
+} from "./collections/methods.js";
+import {
+  backupDatabase,
+  initOrGetBackupStorage,
+  wipeDocumentStorage,
+} from "./storage/methods.js";
+import path from "path";
+import {
+  type AfterImportActions,
+  type ConverterFunctions,
+  converterFunctions,
+  validationRules,
+  type ValidationRules,
+} from "appwrite-utils";
+import { afterImportActions } from "./migrations/afterImportActions.js";
+import {
+  transferDatabaseLocalToLocal,
+  transferDatabaseLocalToRemote,
+  transferStorageLocalToLocal,
+  transferStorageLocalToRemote,
+  type TransferOptions,
+} from "./migrations/transfer.js";
+import { getClient } from "./utils/getClientFromConfig.js";
+import { fetchAllDatabases } from "./migrations/databases.js";
+
+export interface SetupOptions {
+  databases?: Models.Database[];
+  collections?: string[];
+  doBackup?: boolean;
+  wipeDatabase?: boolean;
+  wipeDocumentStorage?: boolean;
+  wipeUsers?: boolean;
+  generateSchemas?: boolean;
+  importData?: boolean;
+  checkDuplicates?: boolean;
+  shouldWriteFile?: boolean;
+}
+
+export class UtilsController {
+  private appwriteFolderPath: string;
+  private appwriteConfigPath: string;
+  public config?: AppwriteConfig;
+  public appwriteServer?: Client;
+  public database?: Databases;
+  public storage?: Storage;
+  public converterDefinitions: ConverterFunctions = converterFunctions;
+  public validityRuleDefinitions: ValidationRules = validationRules;
+  public afterImportActionsDefinitions: AfterImportActions = afterImportActions;
+
+  constructor(currentUserDir: string) {
+    const basePath = currentUserDir;
+    const appwriteConfigFound = findAppwriteConfig(basePath);
+    if (!appwriteConfigFound) {
+      throw new Error("Failed to find appwriteConfig.ts");
+    }
+    this.appwriteConfigPath = appwriteConfigFound;
+    this.appwriteFolderPath = path.dirname(appwriteConfigFound);
+    if (!this.appwriteFolderPath) {
+      throw new Error("Failed to get appwriteFolderPath");
+    }
+  }
+
+  async init() {
+    if (!this.config) {
+      console.log("Initializing appwrite client & loading config...");
+      this.config = await loadConfig(this.appwriteFolderPath);
+      if (!this.config) {
+        throw new Error("Failed to load config");
+      }
+      this.appwriteServer = new Client();
+      this.appwriteServer
+        .setEndpoint(this.config.appwriteEndpoint)
+        .setProject(this.config.appwriteProject)
+        .setKey(this.config.appwriteKey);
+      this.database = new Databases(this.appwriteServer);
+      this.storage = new Storage(this.appwriteServer);
+      this.config.appwriteClient = this.appwriteServer;
+    }
+  }
+
+  async setupMigrationDatabase() {
+    await this.init();
+    if (!this.config) throw new Error("Config not initialized");
+    await setupMigrationDatabase(this.config);
+  }
+
+  async ensureDatabasesExist(databases?: Models.Database[]) {
+    await this.init();
+    if (!this.config) throw new Error("Config not initialized");
+    await this.setupMigrationDatabase();
+    await ensureDatabasesExist(this.config);
+  }
+
+  async getDatabasesByIds(ids: string[]) {
+    await this.init();
+    if (!this.database) throw new Error("Database not initialized");
+    const dbs = await this.database.list([
+      Query.limit(500),
+      Query.equal("$id", ids),
+    ]);
+    return dbs.databases;
+  }
+
+  async wipeOtherDatabases(databasesToKeep: Models.Database[]) {
+    await this.init();
+    if (!this.database) throw new Error("Database not initialized");
+    await wipeOtherDatabases(this.database, databasesToKeep);
+  }
+
+  async wipeUsers() {
+    await this.init();
+    if (!this.config || !this.database)
+      throw new Error("Config or database not initialized");
+    const usersController = new UsersController(this.config, this.database);
+    await usersController.wipeUsers();
+  }
+
+  async backupDatabase(database: Models.Database) {
+    await this.init();
+    if (!this.database || !this.storage || !this.config)
+      throw new Error("Database, storage, or config not initialized");
+    await backupDatabase(
+      this.config,
+      this.database,
+      database.$id,
+      this.storage
+    );
+  }
+
+  async wipeDatabase(database: Models.Database) {
+    await this.init();
+    if (!this.database) throw new Error("Database not initialized");
+    return await wipeDatabase(this.database, database.$id);
+  }
+
+  async wipeDocumentStorage(bucketId: string) {
+    await this.init();
+    if (!this.storage) throw new Error("Storage not initialized");
+    await wipeDocumentStorage(this.storage, bucketId);
+  }
+
+  async createOrUpdateCollectionsForDatabases(databases: Models.Database[]) {
+    await this.init();
+    if (!this.database || !this.config)
+      throw new Error("Database or config not initialized");
+    for (const database of databases) {
+      await this.createOrUpdateCollections(database);
+    }
+  }
+
+  async createOrUpdateCollections(
+    database: Models.Database,
+    deletedCollections?: { collectionId: string; collectionName: string }[]
+  ) {
+    await this.init();
+    if (!this.database || !this.config)
+      throw new Error("Database or config not initialized");
+    await createOrUpdateCollections(
+      this.database,
+      database.$id,
+      this.config,
+      deletedCollections
+    );
+  }
+
+  async generateSchemas() {
+    await this.init();
+    if (!this.config) throw new Error("Config not initialized");
+    await generateSchemas(this.config, this.appwriteFolderPath);
+  }
+
+  async importData(options: SetupOptions) {
+    await this.init();
+    if (!this.config || !this.database || !this.storage)
+      throw new Error("Config, database, or storage not initialized");
+    const importDataActions = new ImportDataActions(
+      this.database,
+      this.storage,
+      this.config,
+      this.converterDefinitions,
+      this.validityRuleDefinitions,
+      this.afterImportActionsDefinitions
+    );
+    const importController = new ImportController(
+      this.config,
+      this.database,
+      this.storage,
+      this.appwriteFolderPath,
+      importDataActions,
+      options
+    );
+    await importController.run();
+  }
+
+  async synchronizeConfigurations(
+    databases?: Models.Database[],
+    config?: AppwriteConfig
+  ) {
+    await this.init();
+    if (!this.storage) throw new Error("Storage not initialized");
+    const configToUse = config || this.config;
+    if (!configToUse) throw new Error("Config not initialized");
+    const appwriteToX = new AppwriteToX(
+      configToUse,
+      this.appwriteFolderPath,
+      this.storage
+    );
+    await appwriteToX.toSchemas(databases);
+  }
+
+  async syncDb() {
+    await this.init();
+    if (!this.database) throw new Error("Database not initialized");
+    const databases = await fetchAllDatabases(this.database);
+    await this.ensureDatabasesExist(databases);
+    await this.createOrUpdateCollectionsForDatabases(databases);
+  }
+
+  getAppwriteFolderPath() {
+    return this.appwriteFolderPath;
+  }
+
+  async transferData(options: TransferOptions): Promise<void> {
+    if (!this.database) {
+      throw new Error(
+        "Database is not initialized, is the config file correct & created?"
+      );
+    }
+
+    let sourceClient = this.database;
+    let targetClient: Databases;
+    let sourceDatabases: Models.Database[];
+    let targetDatabases: Models.Database[];
+
+    if (options.isRemote) {
+      if (
+        !options.transferEndpoint ||
+        !options.transferProject ||
+        !options.transferKey
+      ) {
+        throw new Error("Remote transfer options are missing");
+      }
+
+      const remoteClient = getClient(
+        options.transferEndpoint,
+        options.transferProject,
+        options.transferKey
+      );
+      targetClient = new Databases(remoteClient);
+
+      sourceDatabases = await fetchAllDatabases(sourceClient);
+      targetDatabases = await fetchAllDatabases(targetClient);
+    } else {
+      targetClient = sourceClient;
+      sourceDatabases = targetDatabases = await fetchAllDatabases(sourceClient);
+    }
+
+    // Validate that the provided databases exist in the fetched lists
+    const fromDb = sourceDatabases.find((db) => db.$id === options.fromDb.$id);
+    const targetDb = targetDatabases.find(
+      (db) => db.$id === options.targetDb.$id
+    );
+
+    if (!fromDb || !targetDb) {
+      throw new Error("Source or target database not found");
+    }
+
+    if (options.isRemote) {
+      // Remote transfer
+      await transferDatabaseLocalToRemote(
+        sourceClient,
+        options.transferEndpoint!,
+        options.transferProject!,
+        options.transferKey!,
+        fromDb.$id,
+        targetDb.$id
+      );
+
+      if (this.storage && options.sourceBucket && options.targetBucket) {
+        await transferStorageLocalToRemote(
+          this.storage,
+          options.transferEndpoint!,
+          options.transferProject!,
+          options.transferKey!,
+          options.sourceBucket.$id,
+          options.targetBucket.$id
+        );
+      }
+    } else {
+      // Local transfer
+      await transferDatabaseLocalToLocal(
+        sourceClient,
+        fromDb.$id,
+        targetDb.$id
+      );
+
+      if (this.storage && options.sourceBucket && options.targetBucket) {
+        await transferStorageLocalToLocal(
+          this.storage,
+          options.sourceBucket.$id,
+          options.targetBucket.$id
+        );
+      }
+    }
+
+    console.log("Data transfer completed");
+  }
+}

package/tsconfig.json

@@ -1,37 +1,37 @@
-{
-  "compilerOptions": {
-    // Enable latest features
-    "target": "ESNext",
-    "module": "NodeNext",
-    "lib": ["ESNext"],
-    "moduleDetection": "force",
-    "jsx": "react-jsx",
-    "allowJs": true,
-    "allowArbitraryExtensions": true,
-    "allowSyntheticDefaultImports": true,
-    "resolveJsonModule": true,
-
-    // NodeNext mode
-    "moduleResolution": "NodeNext",
-    "verbatimModuleSyntax": true,
-    "noEmit": false,
-    "emitDeclarationOnly": false,
-
-    // Best practices
-    "strict": true,
-    "skipLibCheck": true,
-    "noFallthroughCasesInSwitch": true,
-
-    // Some stricter flags (disabled by default)
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "noPropertyAccessFromIndexSignature": false,
-    "baseUrl": ".",
-    "esModuleInterop": true,
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "declaration": true
-  },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules", "dist", "src/appwrite", "example/*"]
-}
+{
+  "compilerOptions": {
+    // Enable latest features
+    "target": "ESNext",
+    "module": "NodeNext",
+    "lib": ["ESNext"],
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+    "allowArbitraryExtensions": true,
+    "allowSyntheticDefaultImports": true,
+    "resolveJsonModule": true,
+
+    // NodeNext mode
+    "moduleResolution": "NodeNext",
+    "verbatimModuleSyntax": true,
+    "noEmit": false,
+    "emitDeclarationOnly": false,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false,
+    "baseUrl": ".",
+    "esModuleInterop": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "declaration": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "src/appwrite", "example/*"]
+}