appwrite-utils-cli 1.11.0 → 1.12.0

package/dist/collections/wipeOperations.js

@@ -0,0 +1,306 @@
+import { Databases, Query, } from "node-appwrite";
+import { tryAwaitWithRetry } from "appwrite-utils-helpers";
+import { MessageFormatter, isRetryableError, isCriticalError } from "appwrite-utils-helpers";
+import { ProgressManager } from "../shared/progressManager.js";
+import { delay } from "appwrite-utils-helpers";
+import { chunk } from "es-toolkit";
+import pLimit from "p-limit";
+import { fetchAllCollections } from "./methods.js";
+/**
+ * Optimized streaming deletion of all documents from a collection
+ * Uses memory-efficient pagination instead of loading all documents into memory
+ */
+async function wipeDocumentsFromCollection(database, databaseId, collectionId) {
+    try {
+        // Use streaming deletion pattern - fetch and delete in batches without accumulating
+        const FETCH_BATCH_SIZE = 1000; // How many to fetch per query
+        const DELETE_BATCH_SIZE = 200; // How many to delete concurrently
+        const MAX_CONCURRENT_DELETIONS = 10; // Concurrent deletion operations
+        let totalDeleted = 0;
+        let cursor;
+        let hasMoreDocuments = true;
+        MessageFormatter.info("Starting optimized document deletion...", { prefix: "Wipe" });
+        // Create progress tracker (we'll update the total as we discover more documents)
+        const progress = ProgressManager.create(`delete-${collectionId}`, 1, // Start with 1, will update as we go
+        { title: "Deleting documents" });
+        while (hasMoreDocuments) {
+            // Fetch next batch of documents
+            const queries = [Query.limit(FETCH_BATCH_SIZE)];
+            if (cursor) {
+                queries.push(Query.cursorAfter(cursor));
+            }
+            const response = await database.listDocuments(databaseId, collectionId, queries);
+            const documents = response.documents;
+            if (documents.length === 0) {
+                hasMoreDocuments = false;
+                break;
+            }
+            // Update progress total as we discover more documents
+            if (documents.length === FETCH_BATCH_SIZE) {
+                // There might be more documents, update progress total
+                progress.setTotal(totalDeleted + documents.length + 1000); // Estimate more
+            }
+            MessageFormatter.progress(`Processing batch: ${documents.length} documents (${totalDeleted + documents.length} total so far)`, { prefix: "Wipe" });
+            // Delete this batch using optimized concurrent deletion
+            const documentBatches = chunk(documents, DELETE_BATCH_SIZE);
+            const limit = pLimit(MAX_CONCURRENT_DELETIONS);
+            const deletePromises = documentBatches.map((batch) => limit(async () => {
+                const batchDeletePromises = batch.map(async (doc) => {
+                    try {
+                        await tryAwaitWithRetry(async () => database.deleteDocument(databaseId, collectionId, doc.$id));
+                        totalDeleted++;
+                        progress.update(totalDeleted);
+                    }
+                    catch (error) {
+                        const errorMessage = error.message || String(error);
+                        // Enhanced error handling for document deletion
+                        if (errorMessage.includes("Document with the requested ID could not be found")) {
+                            // Document already deleted, skip silently
+                            totalDeleted++;
+                            progress.update(totalDeleted);
+                        }
+                        else if (isCriticalError(errorMessage)) {
+                            // Critical error, log and rethrow to stop operation
+                            MessageFormatter.error(`Critical error deleting document ${doc.$id}: ${errorMessage}`, error, { prefix: "Wipe" });
+                            throw error;
+                        }
+                        else if (isRetryableError(errorMessage)) {
+                            // Retryable error, will be handled by tryAwaitWithRetry
+                            MessageFormatter.progress(`Retryable error for document ${doc.$id}, will retry`, { prefix: "Wipe" });
+                            totalDeleted++;
+                            progress.update(totalDeleted);
+                        }
+                        else {
+                            // Other non-critical errors, log but continue
+                            MessageFormatter.error(`Failed to delete document ${doc.$id}: ${errorMessage}`, error, { prefix: "Wipe" });
+                            totalDeleted++;
+                            progress.update(totalDeleted);
+                        }
+                    }
+                });
+                await Promise.all(batchDeletePromises);
+            }));
+            await Promise.all(deletePromises);
+            // Set up cursor for next iteration
+            if (documents.length < FETCH_BATCH_SIZE) {
+                hasMoreDocuments = false;
+            }
+            else {
+                cursor = documents[documents.length - 1].$id;
+            }
+            // Small delay between fetch cycles to be respectful to the API
+            await delay(10);
+        }
+        // Update final progress total
+        progress.setTotal(totalDeleted);
+        progress.stop();
+        if (totalDeleted === 0) {
+            MessageFormatter.info("No documents found to delete", { prefix: "Wipe" });
+        }
+        else {
+            MessageFormatter.success(`Successfully deleted ${totalDeleted} documents from collection ${collectionId}`, { prefix: "Wipe" });
+        }
+    }
+    catch (error) {
+        MessageFormatter.error(`Error wiping documents from collection ${collectionId}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Wipe" });
+        throw error;
+    }
+}
+export const wipeDatabase = async (database, databaseId) => {
+    MessageFormatter.info(`Wiping database: ${databaseId}`, { prefix: "Wipe" });
+    const existingCollections = await fetchAllCollections(databaseId, database);
+    let collectionsDeleted = [];
+    if (existingCollections.length === 0) {
+        MessageFormatter.info("No collections to delete", { prefix: "Wipe" });
+        return collectionsDeleted;
+    }
+    const progress = ProgressManager.create(`wipe-db-${databaseId}`, existingCollections.length, { title: "Deleting collections" });
+    let processed = 0;
+    for (const { $id: collectionId, name: name } of existingCollections) {
+        MessageFormatter.progress(`Deleting collection: ${collectionId}`, { prefix: "Wipe" });
+        collectionsDeleted.push({
+            collectionId: collectionId,
+            collectionName: name,
+        });
+        tryAwaitWithRetry(async () => await database.deleteCollection(databaseId, collectionId)); // Try to delete the collection and ignore errors if it doesn't exist or if it's already being deleted
+        processed++;
+        progress.update(processed);
+        await delay(100);
+    }
+    progress.stop();
+    MessageFormatter.success(`Deleted ${collectionsDeleted.length} collections from database`, { prefix: "Wipe" });
+    return collectionsDeleted;
+};
+export const wipeCollection = async (database, databaseId, collectionId) => {
+    const collections = await database.listCollections(databaseId, [
+        Query.equal("$id", collectionId),
+    ]);
+    if (collections.total === 0) {
+        MessageFormatter.warning(`Collection ${collectionId} not found`, { prefix: "Wipe" });
+        return;
+    }
+    const collection = collections.collections[0];
+    await wipeDocumentsFromCollection(database, databaseId, collection.$id);
+};
+// TablesDB helpers for wiping
+export const wipeAllTables = async (adapter, databaseId) => {
+    MessageFormatter.info(`Wiping tables in database: ${databaseId}`, { prefix: 'Wipe' });
+    const res = await adapter.listTables({ databaseId, queries: [Query.limit(500)] });
+    const tables = res.tables || [];
+    const deleted = [];
+    const progress = ProgressManager.create(`wipe-db-${databaseId}`, tables.length, { title: 'Deleting tables' });
+    let processed = 0;
+    for (const t of tables) {
+        try {
+            await adapter.deleteTable({ databaseId, tableId: t.$id });
+            deleted.push({ tableId: t.$id, tableName: t.name });
+        }
+        catch (e) {
+            MessageFormatter.error(`Failed deleting table ${t.$id}`, e instanceof Error ? e : new Error(String(e)), { prefix: 'Wipe' });
+        }
+        processed++;
+        progress.update(processed);
+        await delay(100);
+    }
+    progress.stop();
+    return deleted;
+};
+/**
+ * Optimized deletion of all rows from a table.
+ * Uses bulk deletion when possible, but falls back to individual row deletion
+ * for tables with relationship columns (bulk delete not supported for those).
+ */
+export const wipeTableRows = async (adapter, databaseId, tableId) => {
+    try {
+        // Check if the table has relationship columns — bulk delete is not supported for those
+        const tableInfo = await adapter.getTable({ databaseId, tableId });
+        const columns = tableInfo.data?.columns || [];
+        const hasRelationships = columns.some((col) => col.type === "relationship");
+        const DELETE_BATCH_SIZE = 250;
+        let totalDeleted = 0;
+        let hasMoreRows = true;
+        const progress = ProgressManager.create(`delete-${tableId}`, 1, { title: "Deleting table rows" });
+        if (hasRelationships) {
+            // ── Relationship table: fetch rows then delete individually ──
+            MessageFormatter.info("Table has relationship columns — using individual row deletion (bulk delete not supported)", { prefix: "Wipe" });
+            const FETCH_BATCH_SIZE = 1000;
+            const MAX_CONCURRENT_DELETES = 25;
+            const limit = pLimit(MAX_CONCURRENT_DELETES);
+            // Pipeline: prefetch the first batch, then overlap fetch+delete
+            let pendingRows = [];
+            let totalDiscovered = 0;
+            // Fetch helper — always fetches from the top since we're deleting everything
+            const fetchBatch = async () => {
+                const queries = [Query.limit(FETCH_BATCH_SIZE)];
+                const response = await tryAwaitWithRetry(async () => adapter.listRows({ databaseId, tableId, queries }));
+                return response.rows || response.data || [];
+            };
+            // Kick off the first fetch
+            let nextFetchPromise = fetchBatch();
+            while (hasMoreRows) {
+                // Await the prefetched batch
+                const rows = nextFetchPromise ? await nextFetchPromise : [];
+                nextFetchPromise = null;
+                if (rows.length === 0) {
+                    hasMoreRows = false;
+                    break;
+                }
+                totalDiscovered += rows.length;
+                const isLastBatch = rows.length < FETCH_BATCH_SIZE;
+                if (!isLastBatch) {
+                    progress.setTotal(totalDiscovered + 1000);
+                }
+                else {
+                    progress.setTotal(totalDiscovered);
+                }
+                MessageFormatter.progress(`Fetched ${rows.length} rows (${totalDiscovered} discovered, ${totalDeleted} deleted so far)`, { prefix: "Wipe" });
+                // Start deleting this batch — and prefetch the next one concurrently
+                // (only prefetch if we expect more rows)
+                if (!isLastBatch) {
+                    // Wait a moment before prefetching so the first few deletes free up API capacity
+                    nextFetchPromise = delay(200).then(() => fetchBatch());
+                }
+                // Delete each row with concurrency limit
+                const deletePromises = rows.map((row) => limit(async () => {
+                    try {
+                        await tryAwaitWithRetry(async () => adapter.deleteRow({ databaseId, tableId, id: row.$id }));
+                        totalDeleted++;
+                        progress.update(totalDeleted);
+                    }
+                    catch (error) {
+                        const errorMessage = error.message || String(error);
+                        if (errorMessage.includes("could not be found")) {
+                            totalDeleted++;
+                            progress.update(totalDeleted);
+                        }
+                        else if (isCriticalError(errorMessage)) {
+                            MessageFormatter.error(`Critical error deleting row ${row.$id}: ${errorMessage}`, error, { prefix: "Wipe" });
+                            throw error;
+                        }
+                        else {
+                            MessageFormatter.error(`Failed to delete row ${row.$id}: ${errorMessage}`, error, { prefix: "Wipe" });
+                            totalDeleted++;
+                            progress.update(totalDeleted);
+                        }
+                    }
+                }));
+                await Promise.all(deletePromises);
+                if (isLastBatch) {
+                    hasMoreRows = false;
+                }
+                await delay(10);
+            }
+        }
+        else {
+            // ── No relationships: use fast bulk deletion ──
+            if (!adapter.bulkDeleteRows) {
+                MessageFormatter.error("Bulk deletion not available for this adapter - wipe operation not supported", new Error("bulkDeleteRows not available"), { prefix: "Wipe" });
+                throw new Error("Bulk deletion required for wipe operations");
+            }
+            MessageFormatter.info("Starting optimized table row deletion...", { prefix: "Wipe" });
+            while (hasMoreRows) {
+                try {
+                    const result = await tryAwaitWithRetry(async () => adapter.bulkDeleteRows({
+                        databaseId,
+                        tableId,
+                        rowIds: [],
+                        batchSize: DELETE_BATCH_SIZE
+                    }));
+                    const deletedCount = result.total || 0;
+                    if (deletedCount === 0) {
+                        hasMoreRows = false;
+                        break;
+                    }
+                    totalDeleted += deletedCount;
+                    progress.setTotal(totalDeleted + 100);
+                    progress.update(totalDeleted);
+                    MessageFormatter.progress(`Deleted ${deletedCount} rows (${totalDeleted} total so far)`, { prefix: "Wipe" });
+                    await delay(10);
+                }
+                catch (error) {
+                    const errorMessage = error.message || String(error);
+                    if (isCriticalError(errorMessage)) {
+                        MessageFormatter.error(`Critical error during bulk deletion: ${errorMessage}`, error, { prefix: "Wipe" });
+                        throw error;
+                    }
+                    else {
+                        MessageFormatter.error(`Error during deletion batch: ${errorMessage}`, error, { prefix: "Wipe" });
+                    }
+                }
+            }
+        }
+        // Update final progress total
+        progress.setTotal(totalDeleted);
+        progress.stop();
+        if (totalDeleted === 0) {
+            MessageFormatter.info("No rows found to delete", { prefix: "Wipe" });
+        }
+        else {
+            MessageFormatter.success(`Successfully deleted ${totalDeleted} rows from table ${tableId}`, { prefix: "Wipe" });
+        }
+    }
+    catch (error) {
+        MessageFormatter.error(`Error wiping rows from table ${tableId}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Wipe" });
+        throw error;
+    }
+};

package/dist/databases/methods.d.ts

@@ -0,0 +1,6 @@
+import { Databases, type Models } from "node-appwrite";
+export declare const fetchAllDatabases: (database: Databases) => Promise<Models.Database[]>;
+export declare const wipeDatabase: (database: Databases, databaseId: string) => Promise<{
+    collectionId: string;
+    collectionName: string;
+}[]>;

package/dist/databases/methods.js

@@ -0,0 +1,35 @@
+import { Databases, Query } from "node-appwrite";
+import { delay, tryAwaitWithRetry } from "appwrite-utils-helpers";
+import { fetchAllCollections } from "../collections/methods.js";
+import { MessageFormatter } from "appwrite-utils-helpers";
+export const fetchAllDatabases = async (database) => {
+    const databases = await tryAwaitWithRetry(async () => await database.list([Query.limit(25)]));
+    const allDatabases = databases.databases;
+    if (allDatabases.length === 0)
+        return [];
+    let lastDatabaseId = allDatabases[allDatabases.length - 1].$id;
+    while (databases.databases.length === 25) {
+        const moreDatabases = await database.list([
+            Query.limit(25),
+            Query.cursorAfter(lastDatabaseId),
+        ]);
+        allDatabases.push(...moreDatabases.databases);
+        if (moreDatabases.databases.length < 25)
+            break;
+        lastDatabaseId =
+            moreDatabases.databases[moreDatabases.databases.length - 1].$id;
+    }
+    return allDatabases;
+};
+export const wipeDatabase = async (database, databaseId) => {
+    MessageFormatter.info(`Wiping database: ${databaseId}`, { prefix: "Database" });
+    const existingCollections = await fetchAllCollections(databaseId, database);
+    let collectionsDeleted = [];
+    for (const { $id: collectionId, name } of existingCollections) {
+        MessageFormatter.info(`Deleting collection: ${collectionId}`, { prefix: "Database" });
+        collectionsDeleted.push({ collectionId, collectionName: name });
+        await tryAwaitWithRetry(async () => await database.deleteCollection(databaseId, collectionId));
+        await delay(100);
+    }
+    return collectionsDeleted;
+};

package/dist/databases/setup.d.ts

@@ -0,0 +1,5 @@
+import { Databases, type Models } from "node-appwrite";
+import { type AppwriteConfig } from "appwrite-utils";
+export declare const ensureDatabasesExist: (config: AppwriteConfig, databasesToEnsure?: Models.Database[]) => Promise<void>;
+export declare const wipeOtherDatabases: (database: Databases, databasesToKeep: Models.Database[]) => Promise<void>;
+export declare const ensureCollectionsExist: (config: AppwriteConfig, database: Models.Database, collectionsToEnsure?: Models.Collection[]) => Promise<void>;

package/dist/databases/setup.js

@@ -0,0 +1,45 @@
+import { Databases, Query } from "node-appwrite";
+import { tryAwaitWithRetry } from "appwrite-utils-helpers";
+import {} from "appwrite-utils";
+import { ulid } from "ulidx";
+import { MessageFormatter } from "appwrite-utils-helpers";
+export const ensureDatabasesExist = async (config, databasesToEnsure) => {
+    if (!config.appwriteClient) {
+        throw new Error("Appwrite client is not initialized in the config");
+    }
+    const database = new Databases(config.appwriteClient);
+    const databasesToCreate = databasesToEnsure || config.databases || [];
+    if (!databasesToCreate.length) {
+        MessageFormatter.info("No databases to create");
+        return;
+    }
+    const existingDatabases = await tryAwaitWithRetry(async () => await database.list([Query.limit(500)]));
+    for (const db of databasesToCreate) {
+        if (!existingDatabases.databases.some((d) => d.name === db.name)) {
+            await tryAwaitWithRetry(async () => await database.create(db.$id || ulid(), db.name, true));
+            MessageFormatter.success(`${db.name} database created`);
+        }
+    }
+};
+export const wipeOtherDatabases = async (database, databasesToKeep) => {
+    MessageFormatter.info(`Databases to keep: ${databasesToKeep.map(db => db.name).join(", ")}`);
+    const allDatabases = await tryAwaitWithRetry(async () => await database.list([Query.limit(500)]));
+    for (const db of allDatabases.databases) {
+        if (!databasesToKeep.some((d) => d.name === db.name)) {
+            await tryAwaitWithRetry(async () => await database.delete(db.$id));
+            MessageFormatter.success(`Deleted database: ${db.name}`);
+        }
+    }
+};
+export const ensureCollectionsExist = async (config, database, collectionsToEnsure) => {
+    const databaseClient = new Databases(config.appwriteClient);
+    const collectionsToCreate = collectionsToEnsure ||
+        (config.collections ? config.collections : []);
+    const existingCollections = await tryAwaitWithRetry(async () => await databaseClient.listCollections(database.$id, [Query.limit(500)]));
+    for (const collection of collectionsToCreate) {
+        if (!existingCollections.collections.some((c) => c.name === collection.name)) {
+            await tryAwaitWithRetry(async () => await databaseClient.createCollection(database.$id, ulid(), collection.name, undefined, true, true));
+            MessageFormatter.success(`${collection.name} collection created in ${database.name}`);
+        }
+    }
+};

package/dist/examples/yamlTerminologyExample.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Example demonstrating YAML import/export with dual terminology support
+ *
+ * This example shows how to:
+ * 1. Generate YAML files for both collections and tables
+ * 2. Load and convert between terminologies
+ * 3. Validate terminology consistency
+ * 4. Migrate between formats
+ */
+import { type CollectionCreate } from "appwrite-utils";
+/**
+ * Example 1: Generate YAML templates for both collection and table formats
+ */
+export declare function generateTemplateExamples(outputDir: string): Promise<void>;
+/**
+ * Example 2: Convert existing collection definition to different formats
+ */
+export declare function convertCollectionFormats(collection: CollectionCreate): void;
+/**
+ * Example 3: Load and process YAML files with mixed terminologies
+ */
+export declare function processYamlFiles(baseDir: string): Promise<void>;
+/**
+ * Example 4: Migrate between terminologies
+ */
+export declare function migrateTerminology(baseDir: string): Promise<void>;
+/**
+ * Example 5: Validate terminology consistency
+ */
+export declare function validateTerminology(baseDir: string): Promise<void>;
+/**
+ * Example 6: Import configuration with terminology support
+ */
+export declare function importConfigurationExample(appwriteFolderPath: string): Promise<void>;
+/**
+ * Example 7: Manual terminology conversion
+ */
+export declare function manualTerminologyConversion(): void;
+/**
+ * Main example runner
+ */
+export declare function runYamlTerminologyExamples(outputDir: string): Promise<void>;