appwrite-utils-cli 1.7.6 → 1.7.8

package/src/cli/commands/databaseCommands.ts

@@ -3,6 +3,9 @@ import chalk from "chalk";
 import { join } from "node:path";
 import { MessageFormatter } from "../../shared/messageFormatter.js";
 import { ConfirmationDialogs } from "../../shared/confirmationDialogs.js";
+import { SelectionDialogs } from "../../shared/selectionDialogs.js";
+import type { DatabaseSelection, BucketSelection } from "../../shared/selectionDialogs.js";
+import { logger } from "../../shared/logging.js";
 import { fetchAllDatabases } from "../../databases/methods.js";
 import { listBuckets } from "../../storage/methods.js";
 import { getFunction, downloadLatestFunctionDeployment } from "../../functions/methods.js";
@@ -12,23 +15,41 @@ export const databaseCommands = {
   async syncDb(cli: InteractiveCLI): Promise<void> {
     MessageFormatter.progress("Pushing local configuration to Appwrite...", { prefix: "Database" });
 
-    const databases = await (cli as any).selectDatabases(
-      (cli as any).getLocalDatabases(),
-      chalk.blue("Select local databases to push:"),
-      true
-    );
+    try {
+      // Initialize controller
+      await (cli as any).controller!.init();
 
-    if (!databases.length) {
-      MessageFormatter.warning("No databases selected. Skipping database sync.", { prefix: "Database" });
-      return;
-    }
+      // Get available and configured databases
+      const availableDatabases = await fetchAllDatabases((cli as any).controller!.database!);
+      const configuredDatabases = (cli as any).controller!.config?.databases || [];
 
-    try {
-      // Loop through each database and prompt for collections specific to that database
-      for (const database of databases) {
-        MessageFormatter.info(`\n📦 Configuring push for database: ${database.name}`, { prefix: "Database" });
+      // Get local collections for selection
+      const localCollections = (cli as any).getLocalCollections();
+
+      // Prompt about existing configuration
+      const { syncExisting, modifyConfiguration } = await SelectionDialogs.promptForExistingConfig(configuredDatabases);
+
+      // Select databases
+      const selectedDatabaseIds = await SelectionDialogs.selectDatabases(
+        availableDatabases,
+        configuredDatabases,
+        { showSelectAll: true, allowNewOnly: !syncExisting }
+      );
+
+      if (selectedDatabaseIds.length === 0) {
+        MessageFormatter.warning("No databases selected. Skipping database sync.", { prefix: "Database" });
+        return;
+      }
+
+      // Select tables/collections for each database using the existing method
+      const tableSelectionsMap = new Map<string, string[]>();
+      const availableTablesMap = new Map<string, any[]>();
 
-        const collections = await (cli as any).selectCollectionsAndTables(
+      for (const databaseId of selectedDatabaseIds) {
+        const database = availableDatabases.find(db => db.$id === databaseId)!;
+
+        // Use the existing selectCollectionsAndTables method
+        const selectedCollections = await (cli as any).selectCollectionsAndTables(
           database,
           (cli as any).controller!.database!,
           chalk.blue(`Select collections/tables to push to "${database.name}":`),
@@ -36,19 +57,93 @@ export const databaseCommands = {
           true   // prefer local
         );
 
-        if (collections.length === 0) {
+        // Map selected collections to table IDs
+        const selectedTableIds = selectedCollections.map((c: any) => c.$id || c.id);
+
+        // Store selections
+        tableSelectionsMap.set(databaseId, selectedTableIds);
+        availableTablesMap.set(databaseId, selectedCollections);
+
+        if (selectedCollections.length === 0) {
           MessageFormatter.warning(`No collections selected for database "${database.name}". Skipping.`, { prefix: "Database" });
           continue;
         }
+      }
 
-        // Push selected collections to this specific database
-        await (cli as any).controller!.syncDb([database], collections);
-        MessageFormatter.success(
-          `Pushed ${collections.length} collection(s) to database "${database.name}"`,
-          { prefix: "Database" }
-        );
+      // Ask if user wants to select buckets
+      const { selectBuckets } = await inquirer.prompt([
+        {
+          type: "confirm",
+          name: "selectBuckets",
+          message: "Do you want to select storage buckets to sync as well?",
+          default: false,
+        },
+      ]);
+
+      let bucketSelections: BucketSelection[] = [];
+
+      if (selectBuckets) {
+        // Get available and configured buckets
+        try {
+          const availableBucketsResponse = await listBuckets((cli as any).controller!.storage!);
+          const availableBuckets = availableBucketsResponse.buckets || [];
+          const configuredBuckets = (cli as any).controller!.config?.buckets || [];
+
+          if (availableBuckets.length === 0) {
+            MessageFormatter.warning("No storage buckets available in remote instance.", { prefix: "Database" });
+          } else {
+            // Select buckets using SelectionDialogs
+            const selectedBucketIds = await SelectionDialogs.selectBucketsForDatabases(
+              selectedDatabaseIds,
+              availableBuckets,
+              configuredBuckets,
+              { showSelectAll: true, groupByDatabase: true }
+            );
+
+            if (selectedBucketIds.length > 0) {
+              // Create BucketSelection objects
+              bucketSelections = SelectionDialogs.createBucketSelection(
+                selectedBucketIds,
+                availableBuckets,
+                configuredBuckets,
+                availableDatabases
+              );
+
+              MessageFormatter.info(`Selected ${bucketSelections.length} storage bucket(s)`, { prefix: "Database" });
+            }
+          }
+        } catch (error) {
+          MessageFormatter.warning("Failed to fetch storage buckets. Continuing with databases only.", { prefix: "Database" });
+          logger.warn("Storage bucket fetch failed during syncDb", { error: error instanceof Error ? error.message : String(error) });
+        }
       }
 
+      // Create DatabaseSelection objects
+      const databaseSelections = SelectionDialogs.createDatabaseSelection(
+        selectedDatabaseIds,
+        availableDatabases,
+        tableSelectionsMap,
+        configuredDatabases,
+        availableTablesMap
+      );
+
+      // Show confirmation summary
+      const selectionSummary = SelectionDialogs.createSyncSelectionSummary(
+        databaseSelections,
+        bucketSelections
+      );
+
+      const confirmed = await SelectionDialogs.confirmSyncSelection(selectionSummary);
+
+      if (!confirmed) {
+        MessageFormatter.info("Sync operation cancelled by user", { prefix: "Database" });
+        return;
+      }
+
+      // Perform selective sync using the controller
+      MessageFormatter.progress("Starting selective sync...", { prefix: "Database" });
+      await (cli as any).controller!.selectiveSync(databaseSelections, bucketSelections);
+
       MessageFormatter.success("\n✅ All database configurations pushed successfully!", { prefix: "Database" });
 
       // Then handle functions if requested
@@ -104,23 +199,28 @@ export const databaseCommands = {
         (cli as any).controller!.database!
       );
 
-      // Use the controller's synchronizeConfigurations method which handles collections properly
-      MessageFormatter.progress("Pulling collections and generating collection files...", { prefix: "Collections" });
-      await (cli as any).controller!.synchronizeConfigurations(remoteDatabases);
-
-      // Also configure buckets for any new databases
+      // First, prepare the combined database list for bucket configuration
       const localDatabases = (cli as any).controller!.config?.databases || [];
-      const updatedConfig = await (cli as any).configureBuckets({
+      const allDatabases = [
+        ...localDatabases,
+        ...remoteDatabases.filter(
+          (rd: any) => !localDatabases.some((ld: any) => ld.name === rd.name)
+        ),
+      ];
+
+      // Configure buckets FIRST to get user selections before writing config
+      MessageFormatter.progress("Configuring storage buckets...", { prefix: "Buckets" });
+      const configWithBuckets = await (cli as any).configureBuckets({
         ...(cli as any).controller!.config!,
-        databases: [
-          ...localDatabases,
-          ...remoteDatabases.filter(
-            (rd: any) => !localDatabases.some((ld: any) => ld.name === rd.name)
-          ),
-        ],
+        databases: allDatabases,
       });
 
-      (cli as any).controller!.config = updatedConfig;
+      // Update controller config with bucket selections
+      (cli as any).controller!.config = configWithBuckets;
+
+      // Now synchronize configurations with the updated config that includes bucket selections
+      MessageFormatter.progress("Pulling collections and generating collection files...", { prefix: "Collections" });
+      await (cli as any).controller!.synchronizeConfigurations(remoteDatabases, configWithBuckets);
     }
 
     // Then sync functions

package/src/collections/wipeOperations.ts

@@ -7,7 +7,7 @@ import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
 import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 import { MessageFormatter } from "../shared/messageFormatter.js";
 import { ProgressManager } from "../shared/progressManager.js";
-import { isRetryableError, isBulkNotSupportedError, isCriticalError } from "../shared/errorUtils.js";
+import { isRetryableError, isCriticalError } from "../shared/errorUtils.js";
 import { delay } from "../utils/helperFunctions.js";
 import { chunk } from "es-toolkit";
 import pLimit from "p-limit";
@@ -239,8 +239,8 @@ export const wipeAllTables = async (
 };
 
 /**
- * Optimized streaming deletion of all rows from a table
- * Uses bulk deletion when available, falls back to optimized individual deletion
+ * Optimized deletion of all rows from a table using direct bulk deletion
+ * Uses Query.limit() to delete rows without fetching IDs first
  */
 export const wipeTableRows = async (
   adapter: DatabaseAdapter,
@@ -248,123 +248,78 @@ export const wipeTableRows = async (
   tableId: string
 ): Promise<void> => {
   try {
-    // Configuration for optimized deletion
-    const FETCH_BATCH_SIZE = 1000; // How many to fetch per query
-    const BULK_DELETE_BATCH_SIZE = 500; // How many to bulk delete at once
-    const INDIVIDUAL_DELETE_BATCH_SIZE = 200; // For fallback individual deletion
-    const MAX_CONCURRENT_OPERATIONS = 10; // Concurrent bulk/individual operations
+    // Check if bulk deletion is available
+    if (!adapter.bulkDeleteRows) {
+      MessageFormatter.error(
+        "Bulk deletion not available for this adapter - wipe operation not supported",
+        new Error("bulkDeleteRows not available"),
+        { prefix: "Wipe" }
+      );
+      throw new Error("Bulk deletion required for wipe operations");
+    }
 
+    const DELETE_BATCH_SIZE = 250; // How many rows to delete per batch
     let totalDeleted = 0;
-    let cursor: string | undefined;
     let hasMoreRows = true;
 
     MessageFormatter.info("Starting optimized table row deletion...", { prefix: "Wipe" });
 
-    // Create progress tracker (we'll update the total as we discover more rows)
     const progress = ProgressManager.create(
       `delete-${tableId}`,
-      1, // Start with 1, will update as we go
+      1, // Start with 1, will update as we discover more
       { title: "Deleting table rows" }
     );
 
     while (hasMoreRows) {
-      // Fetch next batch of rows
-      const queries = [Query.limit(FETCH_BATCH_SIZE)];
-      if (cursor) {
-        queries.push(Query.cursorAfter(cursor));
+      try {
+        // Delete next batch using Query.limit() - no fetching needed!
+        const result = await tryAwaitWithRetry(async () =>
+          adapter.bulkDeleteRows!({
+            databaseId,
+            tableId,
+            rowIds: [], // Empty array signals we want to use Query.limit instead
+            batchSize: DELETE_BATCH_SIZE
+          })
+        );
+
+        const deletedCount = (result as any).total || 0;
+
+        if (deletedCount === 0) {
+          hasMoreRows = false;
+          break;
+        }
+
+        totalDeleted += deletedCount;
+        progress.setTotal(totalDeleted + 100); // Estimate more rows exist
+        progress.update(totalDeleted);
+
+        MessageFormatter.progress(
+          `Deleted ${deletedCount} rows (${totalDeleted} total so far)`,
+          { prefix: "Wipe" }
+        );
+
+        // Small delay between batches to be respectful to the API
+        await delay(10);
+
+      } catch (error: any) {
+        const errorMessage = error.message || String(error);
+
+        if (isCriticalError(errorMessage)) {
+          MessageFormatter.error(
+            `Critical error during bulk deletion: ${errorMessage}`,
+            error,
+            { prefix: "Wipe" }
+          );
+          throw error;
+        } else {
+          MessageFormatter.error(
+            `Error during deletion batch: ${errorMessage}`,
+            error,
+            { prefix: "Wipe" }
+          );
+          // Continue trying with next batch
+        }
       }
-
-      const response = await adapter.listRows({ databaseId, tableId, queries });
-      const rows: any[] = (response as any).rows || [];
-
-      if (rows.length === 0) {
-        hasMoreRows = false;
-        break;
-      }
-
-      // Update progress total as we discover more rows
-      if (rows.length === FETCH_BATCH_SIZE) {
-        // There might be more rows, update progress total
-        progress.setTotal(totalDeleted + rows.length + 1000); // Estimate more
-      }
-
-      MessageFormatter.progress(
-        `Processing batch: ${rows.length} rows (${totalDeleted + rows.length} total so far)`,
-        { prefix: "Wipe" }
-      );
-
-      // Try to use bulk deletion first, fall back to individual deletion
-      const rowIds = rows.map((row: any) => row.$id);
-
-// Check if bulk deletion is available and try it first
-      if (adapter.bulkDeleteRows) {
-        try {
-          // Attempt bulk deletion (available in TablesDB)
-          const deletedCount = await tryBulkDeletion(adapter, databaseId, tableId, rowIds, BULK_DELETE_BATCH_SIZE, MAX_CONCURRENT_OPERATIONS);
-          totalDeleted += deletedCount;
-          progress.update(totalDeleted);
-        } catch (bulkError) {
-          // Enhanced error handling: categorize the error and decide on fallback strategy
-          const errorMessage = bulkError instanceof Error ? bulkError.message : String(bulkError);
-
-          if (isRetryableError(errorMessage)) {
-            MessageFormatter.progress(
-              `Bulk deletion encountered retryable error, retrying with individual deletion for ${rows.length} rows`,
-              { prefix: "Wipe" }
-            );
-          } else if (isBulkNotSupportedError(errorMessage)) {
-            MessageFormatter.progress(
-              `Bulk deletion not supported by server, switching to individual deletion for ${rows.length} rows`,
-              { prefix: "Wipe" }
-            );
-          } else {
-            MessageFormatter.progress(
-              `Bulk deletion failed (${errorMessage}), falling back to individual deletion for ${rows.length} rows`,
-              { prefix: "Wipe" }
-            );
-          }
-
-          const deletedCount = await tryIndividualDeletion(
-            adapter,
-            databaseId,
-            tableId,
-            rows,
-            INDIVIDUAL_DELETE_BATCH_SIZE,
-            MAX_CONCURRENT_OPERATIONS,
-            progress,
-            totalDeleted
-          );
-          totalDeleted += deletedCount;
-        }
-      } else {
-        // Bulk deletion not available, use optimized individual deletion
-        MessageFormatter.progress(
-          `Using individual deletion for ${rows.length} rows (bulk deletion not available)`,
-          { prefix: "Wipe" }
-        );
-
-        const deletedCount = await tryIndividualDeletion(
-          adapter,
-          databaseId,
-          tableId,
-          rows,
-          INDIVIDUAL_DELETE_BATCH_SIZE,
-          MAX_CONCURRENT_OPERATIONS,
-          progress,
-          totalDeleted
-        );
-        totalDeleted += deletedCount;
-      }
-
-      // Set up cursor for next iteration
-      if (rows.length < FETCH_BATCH_SIZE) {
-        hasMoreRows = false;
-      } else {
-        cursor = rows[rows.length - 1].$id;
-      }
-
-      // Small delay between fetch cycles to be respectful to the API
-      await delay(10);
     }
 
     // Update final progress total
@@ -389,120 +344,3 @@ export const wipeTableRows = async (
     throw error;
   }
 };
-
-/**
- * Helper function to attempt bulk deletion of row IDs
- */
-async function tryBulkDeletion(
-  adapter: DatabaseAdapter,
-  databaseId: string,
-  tableId: string,
-  rowIds: string[],
-  batchSize: number,
-  maxConcurrent: number
-): Promise<number> {
-  if (!adapter.bulkDeleteRows) {
-    throw new Error("Bulk deletion not available on this adapter");
-  }
-
-  const limit = pLimit(maxConcurrent);
-  const batches = chunk(rowIds, batchSize);
-  let successfullyDeleted = 0;
-
-  const deletePromises = batches.map((batch) =>
-    limit(async () => {
-      try {
-        const result = await tryAwaitWithRetry(async () =>
-          adapter.bulkDeleteRows!({ databaseId, tableId, rowIds: batch })
-        );
-        successfullyDeleted += batch.length; // Assume success if no error thrown
-      } catch (error: any) {
-        const errorMessage = error.message || String(error);
-
-        // Enhanced error handling for bulk deletion
-        if (isCriticalError(errorMessage)) {
-          MessageFormatter.error(
-            `Critical error in bulk deletion batch: ${errorMessage}`,
-            error,
-            { prefix: "Wipe" }
-          );
-          throw error;
-        } else {
-          // For non-critical errors in bulk deletion, re-throw to trigger fallback
-          throw new Error(`Bulk deletion batch failed: ${errorMessage}`);
-        }
-      }
-    })
-  );
-
-  await Promise.all(deletePromises);
-  return successfullyDeleted;
-}
-
-/**
- * Helper function for fallback individual deletion
- */
-async function tryIndividualDeletion(
-  adapter: DatabaseAdapter,
-  databaseId: string,
-  tableId: string,
-  rows: any[],
-  batchSize: number,
-  maxConcurrent: number,
-  progress: any,
-  baseDeleted: number
-): Promise<number> {
-  const limit = pLimit(maxConcurrent);
-  const batches = chunk(rows, batchSize);
-  let processedInBatch = 0;
-  let successfullyDeleted = 0;
-
-  const deletePromises = batches.map((batch) =>
-    limit(async () => {
-      const batchDeletePromises = batch.map(async (row: any) => {
-        try {
-          await tryAwaitWithRetry(async () =>
-            adapter.deleteRow({ databaseId, tableId, id: row.$id })
-          );
-          successfullyDeleted++;
-        } catch (error: any) {
-          const errorMessage = error.message || String(error);
-
-          // Enhanced error handling for row deletion
-          if (errorMessage.includes("Row with the requested ID could not be found")) {
-            // Row already deleted, count as success since it's gone
-            successfullyDeleted++;
-          } else if (isCriticalError(errorMessage)) {
-            // Critical error, log and rethrow to stop operation
-            MessageFormatter.error(
-              `Critical error deleting row ${row.$id}: ${errorMessage}`,
-              error,
-              { prefix: "Wipe" }
-            );
-            throw error;
-          } else if (isRetryableError(errorMessage)) {
-            // Retryable error, will be handled by tryAwaitWithRetry
-            MessageFormatter.progress(
-              `Retryable error for row ${row.$id}, will retry`,
-              { prefix: "Wipe" }
-            );
-          } else {
-            // Other non-critical errors, log but continue
-            MessageFormatter.error(
-              `Failed to delete row ${row.$id}: ${errorMessage}`,
-              error,
-              { prefix: "Wipe" }
-            );
-          }
-        }
-        processedInBatch++;
-        progress.update(baseDeleted + successfullyDeleted);
-      });
-
-      await Promise.all(batchDeletePromises);
-    })
-  );
-
-  await Promise.all(deletePromises);
-  return successfullyDeleted;
-}