@njdamstra/appwrite-utils-cli 1.8.9 → 1.10.0

package/src/collections/transferOperations.ts

@@ -4,12 +4,11 @@ import {
   ID,
   Query,
 } from "node-appwrite";
-import { tryAwaitWithRetry, delay, calculateExponentialBackoff } from "../utils/helperFunctions.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
+import { tryAwaitWithRetry, delay, calculateExponentialBackoff, MessageFormatter } from "@njdamstra/appwrite-utils-helpers";
 import { chunk } from "es-toolkit";
-import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
-import { isLegacyDatabases } from "../utils/typeGuards.js";
-import { getAdapter } from "../utils/getClientFromConfig.js";
+import type { DatabaseAdapter } from "@njdamstra/appwrite-utils-helpers";
+import { isLegacyDatabases } from "@njdamstra/appwrite-utils-helpers";
+import { getAdapter } from "@njdamstra/appwrite-utils-helpers";
 
 /**
  * Transfers all documents from one collection to another in a different database

package/src/collections/wipeOperations.ts

@@ -3,12 +3,11 @@ import {
   Query,
   type Models,
 } from "node-appwrite";
-import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
+import type { DatabaseAdapter } from "@njdamstra/appwrite-utils-helpers";
+import { tryAwaitWithRetry } from "@njdamstra/appwrite-utils-helpers";
+import { MessageFormatter, isRetryableError, isCriticalError } from "@njdamstra/appwrite-utils-helpers";
 import { ProgressManager } from "../shared/progressManager.js";
-import { isRetryableError, isCriticalError } from "../shared/errorUtils.js";
-import { delay } from "../utils/helperFunctions.js";
+import { delay } from "@njdamstra/appwrite-utils-helpers";
 import { chunk } from "es-toolkit";
 import pLimit from "p-limit";
 import { fetchAllCollections } from "./methods.js";
@@ -239,8 +238,9 @@ export const wipeAllTables = async (
 };
 
 /**
- * Optimized deletion of all rows from a table using direct bulk deletion
- * Uses Query.limit() to delete rows without fetching IDs first
+ * Optimized deletion of all rows from a table.
+ * Uses bulk deletion when possible, but falls back to individual row deletion
+ * for tables with relationship columns (bulk delete not supported for those).
  */
 export const wipeTableRows = async (
   adapter: DatabaseAdapter,
@@ -248,76 +248,179 @@ export const wipeTableRows = async (
   tableId: string
 ): Promise<void> => {
   try {
-    // Check if bulk deletion is available
-    if (!adapter.bulkDeleteRows) {
-      MessageFormatter.error(
-        "Bulk deletion not available for this adapter - wipe operation not supported",
-        new Error("bulkDeleteRows not available"),
-        { prefix: "Wipe" }
-      );
-      throw new Error("Bulk deletion required for wipe operations");
-    }
+    // Check if the table has relationship columns — bulk delete is not supported for those
+    const tableInfo = await adapter.getTable({ databaseId, tableId });
+    const columns: any[] = (tableInfo.data as any)?.columns || [];
+    const hasRelationships = columns.some((col: any) => col.type === "relationship");
 
-    const DELETE_BATCH_SIZE = 250; // How many rows to delete per batch
+    const DELETE_BATCH_SIZE = 250;
     let totalDeleted = 0;
     let hasMoreRows = true;
 
-    MessageFormatter.info("Starting optimized table row deletion...", { prefix: "Wipe" });
-
     const progress = ProgressManager.create(
       `delete-${tableId}`,
-      1, // Start with 1, will update as we discover more
+      1,
       { title: "Deleting table rows" }
     );
 
-    while (hasMoreRows) {
-      try {
-        // Delete next batch using Query.limit() - no fetching needed!
-        const result = await tryAwaitWithRetry(async () =>
-          adapter.bulkDeleteRows!({
-            databaseId,
-            tableId,
-            rowIds: [], // Empty array signals we want to use Query.limit instead
-            batchSize: DELETE_BATCH_SIZE
-          })
+    if (hasRelationships) {
+      // ── Relationship table: fetch rows then delete individually ──
+      MessageFormatter.info(
+        "Table has relationship columns — using individual row deletion (bulk delete not supported)",
+        { prefix: "Wipe" }
+      );
+
+      const FETCH_BATCH_SIZE = 1000;
+      const MAX_CONCURRENT_DELETES = 25;
+      const limit = pLimit(MAX_CONCURRENT_DELETES);
+
+      // Pipeline: prefetch the first batch, then overlap fetch+delete
+      let pendingRows: any[] = [];
+      let totalDiscovered = 0;
+
+      // Fetch helper — always fetches from the top since we're deleting everything
+      const fetchBatch = async (): Promise<any[]> => {
+        const queries: any[] = [Query.limit(FETCH_BATCH_SIZE)];
+        const response = await tryAwaitWithRetry(async () =>
+          adapter.listRows({ databaseId, tableId, queries })
         );
+        return (response as any).rows || (response as any).data || [];
+      };
 
-        const deletedCount = (result as any).total || 0;
+      // Kick off the first fetch
+      let nextFetchPromise: Promise<any[]> | null = fetchBatch();
 
-        if (deletedCount === 0) {
+      while (hasMoreRows) {
+        // Await the prefetched batch
+        const rows = nextFetchPromise ? await nextFetchPromise : [];
+        nextFetchPromise = null;
+
+        if (rows.length === 0) {
           hasMoreRows = false;
           break;
         }
 
-        totalDeleted += deletedCount;
-        progress.setTotal(totalDeleted + 100); // Estimate more rows exist
-        progress.update(totalDeleted);
+        totalDiscovered += rows.length;
+        const isLastBatch = rows.length < FETCH_BATCH_SIZE;
+
+        if (!isLastBatch) {
+          progress.setTotal(totalDiscovered + 1000);
+        } else {
+          progress.setTotal(totalDiscovered);
+        }
 
         MessageFormatter.progress(
-          `Deleted ${deletedCount} rows (${totalDeleted} total so far)`,
+          `Fetched ${rows.length} rows (${totalDiscovered} discovered, ${totalDeleted} deleted so far)`,
           { prefix: "Wipe" }
         );
 
-        // Small delay between batches to be respectful to the API
-        await delay(10);
+        // Start deleting this batch — and prefetch the next one concurrently
+        // (only prefetch if we expect more rows)
+        if (!isLastBatch) {
+          // Wait a moment before prefetching so the first few deletes free up API capacity
+          nextFetchPromise = delay(200).then(() => fetchBatch());
+        }
+
+        // Delete each row with concurrency limit
+        const deletePromises = rows.map((row: any) =>
+          limit(async () => {
+            try {
+              await tryAwaitWithRetry(async () =>
+                adapter.deleteRow({ databaseId, tableId, id: row.$id })
+              );
+              totalDeleted++;
+              progress.update(totalDeleted);
+            } catch (error: any) {
+              const errorMessage = error.message || String(error);
+              if (errorMessage.includes("could not be found")) {
+                totalDeleted++;
+                progress.update(totalDeleted);
+              } else if (isCriticalError(errorMessage)) {
+                MessageFormatter.error(
+                  `Critical error deleting row ${row.$id}: ${errorMessage}`,
+                  error,
+                  { prefix: "Wipe" }
+                );
+                throw error;
+              } else {
+                MessageFormatter.error(
+                  `Failed to delete row ${row.$id}: ${errorMessage}`,
+                  error,
+                  { prefix: "Wipe" }
+                );
+                totalDeleted++;
+                progress.update(totalDeleted);
+              }
+            }
+          })
+        );
 
-      } catch (error: any) {
-        const errorMessage = error.message || String(error);
+        await Promise.all(deletePromises);
 
-        if (isCriticalError(errorMessage)) {
-          MessageFormatter.error(
-            `Critical error during bulk deletion: ${errorMessage}`,
-            error,
-            { prefix: "Wipe" }
+        if (isLastBatch) {
+          hasMoreRows = false;
+        }
+
+        await delay(10);
+      }
+    } else {
+      // ── No relationships: use fast bulk deletion ──
+      if (!adapter.bulkDeleteRows) {
+        MessageFormatter.error(
+          "Bulk deletion not available for this adapter - wipe operation not supported",
+          new Error("bulkDeleteRows not available"),
+          { prefix: "Wipe" }
+        );
+        throw new Error("Bulk deletion required for wipe operations");
+      }
+
+      MessageFormatter.info("Starting optimized table row deletion...", { prefix: "Wipe" });
+
+      while (hasMoreRows) {
+        try {
+          const result = await tryAwaitWithRetry(async () =>
+            adapter.bulkDeleteRows!({
+              databaseId,
+              tableId,
+              rowIds: [],
+              batchSize: DELETE_BATCH_SIZE
+            })
           );
-          throw error;
-        } else {
-          MessageFormatter.error(
-            `Error during deletion batch: ${errorMessage}`,
-            error,
+
+          const deletedCount = (result as any).total || 0;
+
+          if (deletedCount === 0) {
+            hasMoreRows = false;
+            break;
+          }
+
+          totalDeleted += deletedCount;
+          progress.setTotal(totalDeleted + 100);
+          progress.update(totalDeleted);
+
+          MessageFormatter.progress(
+            `Deleted ${deletedCount} rows (${totalDeleted} total so far)`,
             { prefix: "Wipe" }
           );
-          // Continue trying with next batch
+
+          await delay(10);
+        } catch (error: any) {
+          const errorMessage = error.message || String(error);
+
+          if (isCriticalError(errorMessage)) {
+            MessageFormatter.error(
+              `Critical error during bulk deletion: ${errorMessage}`,
+              error,
+              { prefix: "Wipe" }
+            );
+            throw error;
+          } else {
+            MessageFormatter.error(
+              `Error during deletion batch: ${errorMessage}`,
+              error,
+              { prefix: "Wipe" }
+            );
+          }
         }
       }
     }

package/src/databases/methods.ts

@@ -1,7 +1,7 @@
 import { Databases, Query, type Models } from "node-appwrite";
-import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { delay, tryAwaitWithRetry } from "@njdamstra/appwrite-utils-helpers";
 import { fetchAllCollections } from "../collections/methods.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
+import { MessageFormatter } from "@njdamstra/appwrite-utils-helpers";
 
 export const fetchAllDatabases = async (
   database: Databases

package/src/databases/setup.ts

@@ -1,8 +1,8 @@
 import { Databases, Query, type Models } from "node-appwrite";
-import { tryAwaitWithRetry } from "../utils/index.js";
+import { tryAwaitWithRetry } from "@njdamstra/appwrite-utils-helpers";
 import { type AppwriteConfig } from "@njdamstra/appwrite-utils";
 import { ulid } from "ulidx";
-import { MessageFormatter } from "../shared/messageFormatter.js";
+import { MessageFormatter } from "@njdamstra/appwrite-utils-helpers";
 
 export const ensureDatabasesExist = async (config: AppwriteConfig, databasesToEnsure?: Models.Database[]) => {
   if (!config.appwriteClient) {

package/src/examples/yamlTerminologyExample.ts

@@ -17,8 +17,8 @@ import {
   usesTableTerminology,
   type YamlTerminologyConfig,
   type YamlCollectionData
-} from "../utils/yamlConverter.js";
-import { createYamlLoader } from "../utils/yamlLoader.js";
+} from "@njdamstra/appwrite-utils-helpers";
+import { createYamlLoader } from "@njdamstra/appwrite-utils-helpers";
 import { YamlImportIntegration } from "../migrations/yaml/YamlImportIntegration.js";
 import { createImportSchemas } from "../migrations/yaml/generateImportSchemas.js";
 import {

package/src/functions/deployments.ts

@@ -15,8 +15,8 @@ import {
   updateFunctionSpecifications,
 } from "./methods.js";
 import ignore from "ignore";
-import { MessageFormatter } from "../shared/messageFormatter.js";
-import { resolveFunctionDirectory, validateFunctionDirectory } from './pathResolution.js';
+import { MessageFormatter } from "@njdamstra/appwrite-utils-helpers";
+import { resolveFunctionDirectory, validateFunctionDirectory } from '@njdamstra/appwrite-utils-helpers';
 
 export const deployFunction = async (
   client: Client,
@@ -150,7 +150,8 @@ export const deployLocalFunction = async (
   client: Client,
   functionName: string,
   functionConfig: AppwriteFunction,
-  functionPath?: string
+  functionPath?: string,
+  configDirPath?: string
 ) => {
   let functionExists = true;
   let functionThatExists: Models.Function;
@@ -160,10 +161,10 @@ export const deployLocalFunction = async (
     functionExists = false;
   }
 
-  const configDirPath = process.cwd(); // TODO: This should be passed from caller
+  const resolvedConfigDir = configDirPath ?? process.cwd();
   const resolvedPath = resolveFunctionDirectory(
     functionName,
-    configDirPath,
+    resolvedConfigDir,
     functionConfig.dirPath,
     functionPath
   );

package/src/functions/fnConfigDiscovery.ts

@@ -3,8 +3,8 @@ import path from 'node:path';
 import yaml from 'js-yaml';
 import { homedir } from 'node:os';
 import { AppwriteFunctionSchema, type AppwriteFunction } from '@njdamstra/appwrite-utils';
-import { shouldIgnoreDirectory } from '../utils/directoryUtils.js';
-import { MessageFormatter } from '../shared/messageFormatter.js';
+import { shouldIgnoreDirectory } from '@njdamstra/appwrite-utils-helpers';
+import { MessageFormatter } from '@njdamstra/appwrite-utils-helpers';
 
 function findGitRoot(startDir: string): string {
   let dir = path.resolve(startDir);

package/src/functions/methods.ts

@@ -13,23 +13,36 @@ import {
   type FunctionScope,
   type Specification,
   type Runtime as AppwriteUtilsRuntime,
+  EventTypeSchema,
 } from "@njdamstra/appwrite-utils";
 import chalk from "chalk";
 import { extract as extractTar } from "tar";
-import { MessageFormatter } from "../shared/messageFormatter.js";
-import { expandTildePath, normalizeFunctionName } from "./pathResolution.js";
+import { MessageFormatter } from "@njdamstra/appwrite-utils-helpers";
+import { expandTildePath, normalizeFunctionName } from '@njdamstra/appwrite-utils-helpers';
 
 /**
  * Validates and filters events array for Appwrite functions
  * - Filters out empty/invalid strings
+ * - Validates against EventTypeSchema
  * - Limits to 100 items maximum (Appwrite limit)
  * - Returns empty array if input is invalid
  */
 const validateEvents = (events?: string[]): string[] => {
   if (!events || !Array.isArray(events)) return [];
-  
+
   return events
-    .filter(event => event && typeof event === 'string' && event.trim().length > 0)
+    .filter(event => {
+      if (!event || typeof event !== 'string' || event.trim().length === 0) {
+        return false;
+      }
+      // Validate against EventTypeSchema
+      const result = EventTypeSchema.safeParse(event);
+      if (!result.success) {
+        MessageFormatter.warning(`Invalid event type "${event}" will be filtered out`, { prefix: "Functions" });
+        return false;
+      }
+      return true;
+    })
     .slice(0, 100);
 };
 

package/src/init.ts

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import inquirer from "inquirer";
 import { createEmptyCollection, setupDirsFiles } from "./utils/setupFiles.js";
-import { MessageFormatter } from "./shared/messageFormatter.js";
+import { MessageFormatter } from "@njdamstra/appwrite-utils-helpers";
 
 MessageFormatter.banner("Appwrite Utils CLI Tool", "For more information, visit https://github.com/njdamstra/AppwriteUtils");
 

package/src/interactiveCLI.ts

@@ -35,9 +35,8 @@ import { join } from "node:path";
 import path from "path";
 import fs from "node:fs";
 import os from "node:os";
-import { MessageFormatter } from "./shared/messageFormatter.js";
+import { MessageFormatter, findYamlConfig } from "@njdamstra/appwrite-utils-helpers";
 import { findAppwriteConfig } from "./utils/loadConfigs.js";
-import { findYamlConfig } from "./config/yamlConfig.js";
 
 // Import command modules
 import { configCommands } from "./cli/commands/configCommands.js";
@@ -46,6 +45,7 @@ import { functionCommands } from "./cli/commands/functionCommands.js";
 import { storageCommands } from "./cli/commands/storageCommands.js";
 import { transferCommands } from "./cli/commands/transferCommands.js";
 import { schemaCommands } from "./cli/commands/schemaCommands.js";
+import { importFileCommands } from "./cli/commands/importFileCommands.js";
 
 enum CHOICES {
   MIGRATE_CONFIG = "🔄 Migrate TypeScript config to YAML (.appwrite structure)",
@@ -67,18 +67,27 @@ enum CHOICES {
   GENERATE_SCHEMAS = "🏗️ Generate schemas",
   GENERATE_CONSTANTS = "📋 Generate cross-language constants (TypeScript, Python, PHP, Dart, etc.)",
   IMPORT_DATA = "📥 Import data",
+  IMPORT_FILE = "📄 Import file (CSV/JSON) directly into a table",
   RELOAD_CONFIG = "🔄 Reload configuration files",
   UPDATE_FUNCTION_SPEC = "⚙️ Update function specifications",
   MANAGE_BUCKETS = "🪣 Manage storage buckets",
   EXIT = "👋 Exit",
 }
 
+export interface InteractiveCLIOptions {
+  useSession?: boolean;
+  sessionCookie?: string;
+}
+
 export class InteractiveCLI {
   private controller: UtilsController | undefined;
   private isUsingTypeScriptConfig: boolean = false;
   private lastSelectedCollectionIds: string[] = [];
+  private options: InteractiveCLIOptions;
 
-  constructor(private currentDir: string) {}
+  constructor(private currentDir: string, options: InteractiveCLIOptions = {}) {
+    this.options = options;
+  }
 
   async run(): Promise<void> {
     MessageFormatter.banner(
@@ -86,7 +95,7 @@ export class InteractiveCLI {
       "Welcome to Appwrite Utils CLI Tool by Zach Handley"
     );
     MessageFormatter.info(
-      "For more information, visit https://github.com/zachhandley/AppwriteUtils"
+      "For more information, visit https://github.com/njdamstra/AppwriteUtils"
     );
 
     // Detect configuration type
@@ -180,6 +189,10 @@ export class InteractiveCLI {
           await this.initControllerIfNeeded();
           await schemaCommands.importData(this);
           break;
+        case CHOICES.IMPORT_FILE:
+          await this.initControllerIfNeeded();
+          await importFileCommands.importFile(this);
+          break;
         case CHOICES.RELOAD_CONFIG:
           await configCommands.reloadConfigWithSessionPreservation(this);
           break;
@@ -204,7 +217,10 @@ export class InteractiveCLI {
   }): Promise<void> {
     if (!this.controller) {
       this.controller = UtilsController.getInstance(this.currentDir, directConfig);
-      await this.controller.init();
+      await this.controller.init({
+        useSession: this.options.useSession,
+        sessionCookie: this.options.sessionCookie
+      });
     } else {
       // Extract session info from existing controller before reinitializing
       const sessionInfo = await this.controller.getSessionInfo();
@@ -219,12 +235,18 @@ export class InteractiveCLI {
         // Reinitialize with session preservation
         UtilsController.clearInstance();
         this.controller = UtilsController.getInstance(this.currentDir, enhancedDirectConfig);
-        await this.controller.init();
+        await this.controller.init({
+          useSession: this.options.useSession,
+          sessionCookie: this.options.sessionCookie
+        });
       } else if (directConfig) {
         // Standard reinitialize without session
         UtilsController.clearInstance();
         this.controller = UtilsController.getInstance(this.currentDir, directConfig);
-        await this.controller.init();
+        await this.controller.init({
+          useSession: this.options.useSession,
+          sessionCookie: this.options.sessionCookie
+        });
       }
       // If no directConfig provided, keep existing controller
     }
@@ -308,7 +330,8 @@ export class InteractiveCLI {
       },
     ]);
 
-    return selectedDatabases;
+    // "list" type returns a single value, "checkbox" returns an array — normalize to array
+    return Array.isArray(selectedDatabases) ? selectedDatabases : [selectedDatabases];
   }
 
   private async selectCollections(
@@ -383,8 +406,14 @@ export class InteractiveCLI {
         (coll) => !configCollections.some((c) => c.name === coll.name || c.$id === coll.$id)
       );
 
+    const getCollectionId = (collection: Models.Collection) => collection.$id || collection.name;
+    const localCollectionIds = new Set(configCollections.map((c) => c.$id || c.name));
+    const localCollections = allCollections.filter((collection) =>
+      localCollectionIds.has(getCollectionId(collection))
+    );
+
     // Enhanced choice display with type indicators
-    const choices = allCollections
+    const choices: { name: string; value: Models.Collection | string }[] = allCollections
       .sort((a, b) => {
         // Sort by type first (collections before tables), then by name
         const aIsTable = (a as any)._isFromTablesDir || false;
@@ -438,6 +467,20 @@ export class InteractiveCLI {
         };
       });
 
+    if (multiSelect && localCollections.length > 1) {
+      choices.unshift({
+        name: chalk.green.bold(`📋 Select All Local Items (${localCollections.length})`),
+        value: "__SELECT_ALL_LOCAL__"
+      });
+    }
+
+    if (multiSelect && allCollections.length > 1) {
+      choices.unshift({
+        name: chalk.green.bold(`📋 Select All Shown (${allCollections.length})`),
+        value: "__SELECT_ALL__"
+      });
+    }
+
     const { selectedCollections } = await inquirer.prompt([
       {
         type: multiSelect ? "checkbox" : "list",
@@ -446,9 +489,28 @@ export class InteractiveCLI {
         choices,
         loop: true,
         pageSize: 15, // Increased page size to accommodate additional info
+        validate: (input: any[]) => {
+          if (!multiSelect) return true;
+          if (input.includes("__SELECT_ALL__") && input.length > 1) {
+            return "Cannot select 'Select All' with individual items.";
+          }
+          if (input.includes("__SELECT_ALL_LOCAL__") && input.length > 1) {
+            return "Cannot select 'Select All Local' with individual items.";
+          }
+          return true;
+        }
       },
     ]);
 
+    if (multiSelect && Array.isArray(selectedCollections)) {
+      if (selectedCollections.includes("__SELECT_ALL__")) {
+        return allCollections;
+      }
+      if (selectedCollections.includes("__SELECT_ALL_LOCAL__")) {
+        return localCollections;
+      }
+    }
+
     return selectedCollections;
   }
 
@@ -1019,7 +1081,10 @@ export class InteractiveCLI {
     _sourceFolder?: string;
     databaseId?: string;
   })[] {
-    const configCollections = this.controller!.config?.collections || [];
+    const configCollections = [
+      ...(this.controller!.config?.collections || []),
+      ...(this.controller!.config?.tables || [])
+    ];
     // @ts-expect-error - appwrite invalid types
     return configCollections.map((c) => ({
       $id: c.$id || ulid(),