@njdamstra/appwrite-utils-cli 1.8.9

package/dist/storage/methods.js

@@ -0,0 +1,472 @@
+import { Compression, Databases, Permission, Query, Role, Storage, } from "node-appwrite";
+import { tryAwaitWithRetry } from "@njdamstra/appwrite-utils";
+import { getClientFromConfig } from "../utils/getClientFromConfig.js";
+import { ulid } from "ulidx";
+import { logOperation } from "../shared/operationLogger.js";
+import { splitIntoBatches } from "../shared/migrationHelpers.js";
+import { retryFailedPromises } from "../utils/retryFailedPromises.js";
+import { InputFile } from "node-appwrite/file";
+import { MessageFormatter, Messages } from "../shared/messageFormatter.js";
+import { ProgressManager } from "../shared/progressManager.js";
+import { recordBackup } from "../shared/backupTracking.js";
+import { AdapterFactory } from "../adapters/AdapterFactory.js";
+import { createBackupZip } from "./backupCompression.js";
+export const getStorage = (config) => {
+    const client = getClientFromConfig(config);
+    return new Storage(client);
+};
+export const listBuckets = async (storage, queries, search) => {
+    return await storage.listBuckets(queries, search);
+};
+export const getBucket = async (storage, bucketId) => {
+    return await storage.getBucket(bucketId);
+};
+export const createBucket = async (storage, bucket, bucketId) => {
+    return await storage.createBucket(bucketId ?? ulid(), bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, bucket.maximumFileSize, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
+};
+export const updateBucket = async (storage, bucket, bucketId) => {
+    return await storage.updateBucket(bucketId, bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, bucket.maximumFileSize, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
+};
+export const deleteBucket = async (storage, bucketId) => {
+    return await storage.deleteBucket(bucketId);
+};
+export const getFile = async (storage, bucketId, fileId) => {
+    return await storage.getFile(bucketId, fileId);
+};
+export const listFiles = async (storage, bucketId, queries, search) => {
+    return await storage.listFiles(bucketId, queries, search);
+};
+export const deleteFile = async (storage, bucketId, fileId) => {
+    return await storage.deleteFile(bucketId, fileId);
+};
+export const ensureDatabaseConfigBucketsExist = async (storage, config, databases = []) => {
+    for (const db of databases) {
+        const database = config.databases?.find((d) => d.$id === db.$id);
+        if (database?.bucket) {
+            try {
+                const existing = await storage.getBucket(database.bucket.$id);
+                // Compare and update if needed
+                const desired = database.bucket;
+                // Build desired permissions via Permission helper
+                const permissions = [];
+                if (desired.permissions && desired.permissions.length > 0) {
+                    for (const p of desired.permissions) {
+                        switch (p.permission) {
+                            case 'read':
+                                permissions.push(Permission.read(p.target));
+                                break;
+                            case 'create':
+                                permissions.push(Permission.create(p.target));
+                                break;
+                            case 'update':
+                                permissions.push(Permission.update(p.target));
+                                break;
+                            case 'delete':
+                                permissions.push(Permission.delete(p.target));
+                                break;
+                            case 'write':
+                                permissions.push(Permission.write(p.target));
+                                break;
+                            default: break;
+                        }
+                    }
+                }
+                const diff = (existing.name !== desired.name ||
+                    JSON.stringify(existing.$permissions || []) !== JSON.stringify(permissions) ||
+                    !!existing.fileSecurity !== !!desired.fileSecurity ||
+                    !!existing.enabled !== !!desired.enabled ||
+                    (existing.maximumFileSize ?? undefined) !== (desired.maximumFileSize ?? undefined) ||
+                    JSON.stringify(existing.allowedFileExtensions || []) !== JSON.stringify(desired.allowedFileExtensions || []) ||
+                    String(existing.compression || 'none') !== String(desired.compression || 'none') ||
+                    !!existing.encryption !== !!desired.encryption ||
+                    !!existing.antivirus !== !!desired.antivirus);
+                if (diff) {
+                    try {
+                        await storage.updateBucket(desired.$id, desired.name, permissions, desired.fileSecurity, desired.enabled, desired.maximumFileSize, desired.allowedFileExtensions, desired.compression, desired.encryption, desired.antivirus);
+                        MessageFormatter.info(`Updated bucket ${desired.$id} to match config`, { prefix: 'Buckets' });
+                    }
+                    catch (updateErr) {
+                        MessageFormatter.warning(`Failed to update bucket ${desired.$id}: ${updateErr instanceof Error ? updateErr.message : String(updateErr)}`, { prefix: 'Buckets' });
+                    }
+                }
+                else {
+                    MessageFormatter.debug(`Bucket ${desired.$id} up-to-date`, undefined, { prefix: 'Buckets' });
+                }
+            }
+            catch (e) {
+                const permissions = [];
+                if (database.bucket.permissions &&
+                    database.bucket.permissions.length > 0) {
+                    for (const permission of database.bucket.permissions) {
+                        switch (permission.permission) {
+                            case "read":
+                                permissions.push(Permission.read(permission.target));
+                                break;
+                            case "create":
+                                permissions.push(Permission.create(permission.target));
+                                break;
+                            case "update":
+                                permissions.push(Permission.update(permission.target));
+                                break;
+                            case "delete":
+                                permissions.push(Permission.delete(permission.target));
+                                break;
+                            case "write":
+                                permissions.push(Permission.write(permission.target));
+                                break;
+                            default:
+                                console.warn(`Unknown permission: ${permission.permission}`);
+                                break;
+                        }
+                    }
+                }
+                try {
+                    await storage.createBucket(database.bucket.$id, database.bucket.name, permissions, database.bucket.fileSecurity, database.bucket.enabled, database.bucket.maximumFileSize, database.bucket.allowedFileExtensions, database.bucket.compression, database.bucket.encryption, database.bucket.antivirus);
+                    MessageFormatter.success(`Bucket ${database.bucket.$id} created`, { prefix: 'Buckets' });
+                }
+                catch (createError) {
+                    MessageFormatter.error(`Failed to create bucket ${database.bucket.$id}`, createError instanceof Error ? createError : new Error(String(createError)), { prefix: 'Buckets' });
+                }
+            }
+        }
+    }
+};
+export const wipeDocumentStorage = async (storage, bucketId, options = {}) => {
+    MessageFormatter.warning(`About to delete all files in bucket: ${bucketId}`);
+    if (!options.skipConfirmation) {
+        const { ConfirmationDialogs } = await import("../shared/confirmationDialogs.js");
+        const confirmed = await ConfirmationDialogs.confirmDestructiveOperation({
+            operation: "Storage Wipe",
+            targets: [bucketId],
+            consequences: [
+                "Delete ALL files in the storage bucket",
+                "This action cannot be undone",
+            ],
+            requireExplicitConfirmation: true,
+            confirmationText: "DELETE FILES",
+        });
+        if (!confirmed) {
+            MessageFormatter.info("Storage wipe cancelled by user");
+            return;
+        }
+    }
+    MessageFormatter.progress(`Scanning files in bucket: ${bucketId}`);
+    let moreFiles = true;
+    let lastFileId;
+    const allFiles = [];
+    // First pass: collect all file IDs
+    while (moreFiles) {
+        const queries = [Query.limit(100)];
+        if (lastFileId) {
+            queries.push(Query.cursorAfter(lastFileId));
+        }
+        const filesPulled = await tryAwaitWithRetry(async () => await storage.listFiles(bucketId, queries));
+        if (filesPulled.files.length === 0) {
+            moreFiles = false;
+            break;
+        }
+        else if (filesPulled.files.length > 0) {
+            const fileIds = filesPulled.files.map((file) => file.$id);
+            allFiles.push(...fileIds);
+        }
+        moreFiles = filesPulled.files.length === 100;
+        if (moreFiles) {
+            lastFileId = filesPulled.files[filesPulled.files.length - 1].$id;
+        }
+    }
+    if (allFiles.length === 0) {
+        MessageFormatter.info("No files found in bucket");
+        return;
+    }
+    // Second pass: delete files with progress tracking
+    const progress = ProgressManager.create(`wipe-${bucketId}`, allFiles.length, {
+        title: `Deleting files from ${bucketId}`,
+    });
+    try {
+        for (let i = 0; i < allFiles.length; i++) {
+            const fileId = allFiles[i];
+            await tryAwaitWithRetry(async () => await storage.deleteFile(bucketId, fileId));
+            progress.update(i + 1, `Deleted file: ${fileId.slice(0, 20)}...`);
+        }
+        progress.stop();
+        MessageFormatter.success(`All ${MessageFormatter.formatNumber(allFiles.length)} files in bucket ${bucketId} have been deleted`);
+    }
+    catch (error) {
+        progress.fail(error instanceof Error ? error.message : String(error));
+        throw error;
+    }
+};
+export const initOrGetDocumentStorage = async (storage, config, dbId, bucketName) => {
+    const bucketId = bucketName ??
+        `${config.documentBucketId}_${dbId.toLowerCase().replace(" ", "")}`;
+    try {
+        return await tryAwaitWithRetry(async () => await storage.getBucket(bucketId));
+    }
+    catch (e) {
+        return await tryAwaitWithRetry(async () => await storage.createBucket(bucketId, `${dbId} Storage`, [
+            Permission.read(Role.any()),
+            Permission.create(Role.users()),
+            Permission.update(Role.users()),
+            Permission.delete(Role.users()),
+        ]));
+    }
+};
+/**
+ * Initializes or gets the centralized backup bucket
+ * All backups are stored in a single "appwrite-backups" bucket
+ */
+export const initBackupBucket = async (storage) => {
+    const BACKUP_BUCKET_ID = "appwrite-backups";
+    const BACKUP_BUCKET_NAME = "Backups";
+    try {
+        // Try to get existing bucket
+        const bucket = await storage.getBucket(BACKUP_BUCKET_ID);
+        return bucket;
+    }
+    catch (error) {
+        // Bucket doesn't exist, create it
+        try {
+            const bucket = await storage.createBucket(BACKUP_BUCKET_ID, BACKUP_BUCKET_NAME, [
+                Permission.read(Role.any()),
+                Permission.create(Role.users()),
+                Permission.update(Role.users()),
+                Permission.delete(Role.users())
+            ], false, // fileSecurity
+            true, // enabled
+            undefined, // maximumFileSize
+            undefined, // allowedFileExtensions
+            Compression.Gzip, // compression
+            false, // encryption
+            false // antivirus
+            );
+            MessageFormatter.success(`Created backup bucket: ${BACKUP_BUCKET_ID}`);
+            return bucket;
+        }
+        catch (createError) {
+            MessageFormatter.error("Failed to create backup bucket", createError instanceof Error ? createError : new Error(String(createError)));
+            return undefined;
+        }
+    }
+};
+export const backupDatabase = async (config, database, databaseId, storage, format = 'json') => {
+    const startTime = Date.now();
+    MessageFormatter.banner("Database Backup", `Backing up database: ${databaseId}`);
+    MessageFormatter.info(Messages.BACKUP_STARTED(databaseId));
+    let data = {
+        database: "",
+        collections: [],
+        documents: [],
+    };
+    const backupOperation = await logOperation(database, databaseId, {
+        operationType: "backup",
+        collectionId: "",
+        data: "Starting backup...",
+        progress: 0,
+        total: 100,
+        error: "",
+        status: "in_progress",
+    });
+    let progress = null;
+    let totalDocuments = 0;
+    let processedDocuments = 0;
+    try {
+        const db = await tryAwaitWithRetry(async () => await database.get(databaseId));
+        data.database = JSON.stringify(db);
+        // First pass: count collections and documents for progress tracking
+        MessageFormatter.step(1, 3, "Analyzing database structure");
+        let lastCollectionId = "";
+        let moreCollections = true;
+        let totalCollections = 0;
+        while (moreCollections) {
+            const collectionResponse = await tryAwaitWithRetry(async () => await database.listCollections(databaseId, [
+                Query.limit(500),
+                ...(lastCollectionId ? [Query.cursorAfter(lastCollectionId)] : []),
+            ]));
+            totalCollections += collectionResponse.collections.length;
+            // Count documents in each collection
+            for (const { $id: collectionId } of collectionResponse.collections) {
+                try {
+                    const documentCount = await tryAwaitWithRetry(async () => (await database.listDocuments(databaseId, collectionId, [Query.limit(1)])).total);
+                    totalDocuments += documentCount;
+                }
+                catch (error) {
+                    MessageFormatter.warning(`Could not count documents in collection ${collectionId}`);
+                }
+            }
+            moreCollections = collectionResponse.collections.length === 500;
+            if (moreCollections) {
+                lastCollectionId = collectionResponse.collections[collectionResponse.collections.length - 1].$id;
+            }
+        }
+        const totalItems = totalCollections + totalDocuments;
+        progress = ProgressManager.create(`backup-${databaseId}`, totalItems, {
+            title: `Backing up ${databaseId}`,
+        });
+        MessageFormatter.step(2, 3, `Processing ${totalCollections} collections and ${totalDocuments} documents`);
+        // Second pass: actual backup with progress tracking
+        lastCollectionId = "";
+        moreCollections = true;
+        while (moreCollections) {
+            const collectionResponse = await tryAwaitWithRetry(async () => await database.listCollections(databaseId, [
+                Query.limit(500),
+                ...(lastCollectionId ? [Query.cursorAfter(lastCollectionId)] : []),
+            ]));
+            for (const { $id: collectionId, name: collectionName, } of collectionResponse.collections) {
+                try {
+                    const collection = await tryAwaitWithRetry(async () => await database.getCollection(databaseId, collectionId));
+                    data.collections.push(JSON.stringify(collection));
+                    progress?.increment(1, `Processing collection: ${collectionName}`);
+                    let lastDocumentId = "";
+                    let moreDocuments = true;
+                    let collectionDocumentCount = 0;
+                    while (moreDocuments) {
+                        const documentResponse = await tryAwaitWithRetry(async () => await database.listDocuments(databaseId, collectionId, [
+                            Query.limit(500),
+                            ...(lastDocumentId
+                                ? [Query.cursorAfter(lastDocumentId)]
+                                : []),
+                        ]));
+                        collectionDocumentCount += documentResponse.documents.length;
+                        const documentPromises = documentResponse.documents.map(({ $id: documentId }) => database.getDocument(databaseId, collectionId, documentId));
+                        const promiseBatches = splitIntoBatches(documentPromises);
+                        const documentsPulled = [];
+                        for (const batch of promiseBatches) {
+                            const successfulDocuments = await retryFailedPromises(batch);
+                            documentsPulled.push(...successfulDocuments);
+                            // Update progress for each batch
+                            progress?.increment(successfulDocuments.length, `Processing ${collectionName}: ${processedDocuments + successfulDocuments.length}/${totalDocuments} documents`);
+                            processedDocuments += successfulDocuments.length;
+                        }
+                        data.documents.push({
+                            collectionId: collectionId,
+                            data: JSON.stringify(documentsPulled),
+                        });
+                        if (backupOperation) {
+                            await logOperation(database, databaseId, {
+                                operationType: "backup",
+                                collectionId: collectionId,
+                                data: `Backing up, ${data.collections.length} collections so far`,
+                                progress: processedDocuments,
+                                total: totalDocuments,
+                                error: "",
+                                status: "in_progress",
+                            }, backupOperation.$id);
+                        }
+                        moreDocuments = documentResponse.documents.length === 500;
+                        if (moreDocuments) {
+                            lastDocumentId =
+                                documentResponse.documents[documentResponse.documents.length - 1].$id;
+                        }
+                    }
+                    MessageFormatter.success(`Collection ${collectionName} backed up with ${MessageFormatter.formatNumber(collectionDocumentCount)} documents`);
+                }
+                catch (error) {
+                    MessageFormatter.warning(`Collection ${collectionName} could not be backed up: ${error instanceof Error ? error.message : String(error)}`);
+                    continue;
+                }
+            }
+            moreCollections = collectionResponse.collections.length === 500;
+            if (moreCollections) {
+                lastCollectionId =
+                    collectionResponse.collections[collectionResponse.collections.length - 1].$id;
+            }
+        }
+        MessageFormatter.step(3, 3, "Creating backup file");
+        const bucket = await initBackupBucket(storage);
+        if (!bucket) {
+            throw new Error("Failed to initialize backup bucket");
+        }
+        let inputFile;
+        let fileName;
+        let backupSize;
+        if (format === 'zip') {
+            // Create compressed backup
+            const zipBuffer = await createBackupZip(data);
+            fileName = `${new Date().toISOString()}-${databaseId}.zip`;
+            backupSize = zipBuffer.length;
+            inputFile = InputFile.fromBuffer(new Uint8Array(zipBuffer), fileName);
+        }
+        else {
+            // Use JSON format (existing logic)
+            const backupData = JSON.stringify(data, null, 2);
+            fileName = `${new Date().toISOString()}-${databaseId}.json`;
+            backupSize = Buffer.byteLength(backupData, 'utf8');
+            inputFile = InputFile.fromPlainText(backupData, fileName);
+        }
+        const fileCreated = await storage.createFile(bucket.$id, ulid(), inputFile);
+        progress?.stop();
+        // Record backup metadata
+        try {
+            const { adapter } = await AdapterFactory.create({
+                appwriteEndpoint: config.appwriteEndpoint,
+                appwriteProject: config.appwriteProject,
+                appwriteKey: config.appwriteKey,
+                sessionCookie: config.sessionCookie
+            });
+            await recordBackup(adapter, databaseId, {
+                backupId: fileCreated.$id,
+                backupType: 'database',
+                databaseId: databaseId,
+                sizeBytes: backupSize,
+                collections: data.collections.length,
+                documents: processedDocuments,
+                format: format,
+                status: 'completed',
+                restorationStatus: 'not_restored'
+            });
+        }
+        catch (metadataError) {
+            // Don't fail backup if metadata recording fails
+            MessageFormatter.warning(`Failed to record backup metadata: ${metadataError instanceof Error ? metadataError.message : String(metadataError)}`);
+        }
+        if (backupOperation) {
+            await logOperation(database, databaseId, {
+                operationType: "backup",
+                collectionId: "",
+                data: fileCreated.$id,
+                progress: totalItems,
+                total: totalItems,
+                error: "",
+                status: "completed",
+            }, backupOperation.$id);
+        }
+        const duration = Date.now() - startTime;
+        MessageFormatter.operationSummary("Backup", {
+            database: databaseId,
+            collections: data.collections.length,
+            documents: processedDocuments,
+            fileSize: MessageFormatter.formatBytes(backupSize),
+            backupFile: fileName,
+            bucket: bucket.$id,
+        }, duration);
+        MessageFormatter.success(Messages.BACKUP_COMPLETED(databaseId, backupSize));
+        // Return backup result for tracking
+        const dbData = JSON.parse(data.database);
+        return {
+            backupFileId: fileCreated.$id,
+            backupFileName: fileName,
+            backupSizeBytes: backupSize,
+            databaseId: databaseId,
+            databaseName: dbData.name || databaseId,
+            collectionCount: data.collections.length,
+            documentCount: processedDocuments,
+            format: format
+        };
+    }
+    catch (error) {
+        progress?.fail(error instanceof Error ? error.message : String(error));
+        MessageFormatter.error("Backup failed", error instanceof Error ? error : new Error(String(error)));
+        if (backupOperation) {
+            await logOperation(database, databaseId, {
+                operationType: "backup",
+                collectionId: "",
+                data: "Backup failed",
+                progress: 0,
+                total: totalDocuments,
+                error: String(error),
+                status: "error",
+            }, backupOperation.$id);
+        }
+        throw error;
+    }
+};