appwrite-utils-cli 1.5.2 → 1.6.0

package/dist/backups/operations/bucketBackup.js

@@ -0,0 +1,197 @@
+import JSZip from "jszip";
+import { ID, Query } from "node-appwrite";
+import { InputFile } from "node-appwrite/file";
+import pLimit from "p-limit";
+import { MessageFormatter } from "../../shared/messageFormatter.js";
+import { logger } from "../../shared/logging.js";
+import { ulid } from "ulidx";
+/**
+ * Downloads all files from a bucket in parallel and creates a ZIP backup
+ */
+export async function backupBucket(storage, bucketId, backupBucketId, options = {}) {
+    const { compressionLevel = 6, parallelDownloads = 10, onProgress } = options;
+    const errors = [];
+    let totalSizeBytes = 0;
+    try {
+        // Step 1: Get bucket metadata
+        MessageFormatter.info(`Fetching bucket metadata for ${bucketId}`, { prefix: "Backup" });
+        const bucket = await storage.getBucket(bucketId);
+        // Step 2: List all files in bucket with pagination
+        MessageFormatter.info(`Listing all files in bucket ${bucketId}`, { prefix: "Backup" });
+        const allFiles = [];
+        let lastFileId;
+        while (true) {
+            const queries = [Query.limit(100)];
+            if (lastFileId) {
+                queries.push(Query.cursorAfter(lastFileId));
+            }
+            const filesPage = await storage.listFiles(bucketId, queries);
+            allFiles.push(...filesPage.files);
+            if (filesPage.files.length < 100)
+                break;
+            lastFileId = filesPage.files[filesPage.files.length - 1].$id;
+        }
+        MessageFormatter.info(`Found ${allFiles.length} files to backup`, { prefix: "Backup" });
+        if (allFiles.length === 0) {
+            // Empty bucket - create minimal backup
+            const manifest = {
+                version: "1.0",
+                bucketId: bucket.$id,
+                bucketName: bucket.name,
+                createdAt: new Date().toISOString(),
+                fileCount: 0,
+                totalSizeBytes: 0,
+                compression: bucket.compression === 'gzip' ? 'gzip' : 'none',
+                files: [],
+                bucketConfiguration: {
+                    $permissions: bucket.$permissions,
+                    fileSecurity: bucket.fileSecurity,
+                    enabled: bucket.enabled,
+                    maximumFileSize: bucket.maximumFileSize,
+                    allowedFileExtensions: bucket.allowedFileExtensions,
+                    compression: bucket.compression,
+                    encryption: bucket.encryption,
+                    antivirus: bucket.antivirus
+                }
+            };
+            const manifestFileId = await uploadManifest(storage, backupBucketId, bucketId, manifest);
+            return {
+                backupFileId: '',
+                manifestFileId,
+                fileCount: 0,
+                totalSizeBytes: 0,
+                zipSizeBytes: 0,
+                status: 'completed'
+            };
+        }
+        // Step 3: Download all files in parallel with concurrency limit
+        MessageFormatter.info(`Downloading ${allFiles.length} files in parallel (max ${parallelDownloads} concurrent)`, { prefix: "Backup" });
+        const limit = pLimit(parallelDownloads);
+        const downloadedFiles = new Map();
+        let successCount = 0;
+        let errorCount = 0;
+        const downloadTasks = allFiles.map((file, index) => limit(async () => {
+            try {
+                const fileBuffer = await storage.getFileDownload(bucketId, file.$id);
+                const buffer = Buffer.from(fileBuffer);
+                downloadedFiles.set(file.$id, { buffer, file });
+                successCount++;
+                totalSizeBytes += file.sizeOriginal || buffer.length;
+                if (onProgress) {
+                    onProgress(successCount + errorCount, allFiles.length, file.name);
+                }
+                logger.debug(`Downloaded file ${file.name}`, {
+                    fileId: file.$id,
+                    size: buffer.length
+                });
+            }
+            catch (error) {
+                errorCount++;
+                const errorMsg = `Failed to download file ${file.name} (${file.$id}): ${error instanceof Error ? error.message : String(error)}`;
+                errors.push(errorMsg);
+                logger.error(errorMsg);
+                if (onProgress) {
+                    onProgress(successCount + errorCount, allFiles.length, file.name);
+                }
+            }
+        }));
+        await Promise.all(downloadTasks);
+        if (successCount === 0) {
+            throw new Error(`Failed to download any files from bucket ${bucketId}`);
+        }
+        MessageFormatter.info(`Successfully downloaded ${successCount}/${allFiles.length} files`, { prefix: "Backup" });
+        // Step 4: Create ZIP archive with all files
+        MessageFormatter.info(`Creating ZIP archive for bucket ${bucketId}`, { prefix: "Backup" });
+        const zip = new JSZip();
+        for (const [fileId, { buffer, file }] of downloadedFiles.entries()) {
+            // Preserve file structure by using file name
+            const fileName = file.name || `file_${fileId}`;
+            zip.file(fileName, new Uint8Array(buffer));
+        }
+        // Generate ZIP buffer
+        const zipBuffer = await zip.generateAsync({
+            type: "nodebuffer",
+            compression: "DEFLATE",
+            compressionOptions: { level: compressionLevel }
+        });
+        const zipSizeBytes = zipBuffer.length;
+        MessageFormatter.info(`ZIP archive created: ${MessageFormatter.formatBytes(zipSizeBytes)}`, { prefix: "Backup" });
+        // Step 5: Create manifest
+        const fileMetadata = Array.from(downloadedFiles.values()).map(({ file }) => ({
+            $id: file.$id,
+            name: file.name,
+            size: file.sizeOriginal,
+            mimeType: file.mimeType,
+            $permissions: file.$permissions,
+            chunksCount: file.chunksTotal,
+            signature: file.signature,
+            $createdAt: file.$createdAt,
+            $updatedAt: file.$updatedAt
+        }));
+        const manifest = {
+            version: "1.0",
+            bucketId: bucket.$id,
+            bucketName: bucket.name,
+            createdAt: new Date().toISOString(),
+            fileCount: successCount,
+            totalSizeBytes,
+            compression: bucket.compression === 'gzip' ? 'gzip' : 'none',
+            files: fileMetadata,
+            bucketConfiguration: {
+                $permissions: bucket.$permissions,
+                fileSecurity: bucket.fileSecurity,
+                enabled: bucket.enabled,
+                maximumFileSize: bucket.maximumFileSize,
+                allowedFileExtensions: bucket.allowedFileExtensions,
+                compression: bucket.compression,
+                encryption: bucket.encryption,
+                antivirus: bucket.antivirus
+            }
+        };
+        // Step 6: Upload backup ZIP to backup bucket
+        MessageFormatter.info(`Uploading backup ZIP to bucket ${backupBucketId}`, { prefix: "Backup" });
+        const backupFileName = `${bucketId}.zip`;
+        const backupFile = await storage.createFile(backupBucketId, ID.unique(), InputFile.fromBuffer(new Uint8Array(zipBuffer), backupFileName));
+        // Step 7: Upload manifest JSON
+        const manifestFileId = await uploadManifest(storage, backupBucketId, bucketId, manifest);
+        const status = errorCount === 0 ? 'completed' :
+            successCount > 0 ? 'partial' :
+                'failed';
+        MessageFormatter.success(`Bucket backup ${status}: ${successCount}/${allFiles.length} files backed up`, { prefix: "Backup" });
+        return {
+            backupFileId: backupFile.$id,
+            manifestFileId,
+            fileCount: successCount,
+            totalSizeBytes,
+            zipSizeBytes,
+            status,
+            errors: errors.length > 0 ? errors : undefined
+        };
+    }
+    catch (error) {
+        const errorMsg = `Bucket backup failed: ${error instanceof Error ? error.message : String(error)}`;
+        MessageFormatter.error(errorMsg, error instanceof Error ? error : new Error(errorMsg), { prefix: "Backup" });
+        return {
+            backupFileId: '',
+            manifestFileId: '',
+            fileCount: 0,
+            totalSizeBytes: 0,
+            zipSizeBytes: 0,
+            status: 'failed',
+            errors: [errorMsg, ...errors]
+        };
+    }
+}
+/**
+ * Uploads bucket manifest JSON to backup bucket
+ */
+async function uploadManifest(storage, backupBucketId, bucketId, manifest) {
+    const manifestFileName = `${bucketId}.json`;
+    const manifestBuffer = Buffer.from(JSON.stringify(manifest, null, 2), 'utf-8');
+    const manifestFile = await storage.createFile(backupBucketId, ID.unique(), InputFile.fromBuffer(new Uint8Array(manifestBuffer), manifestFileName));
+    logger.info("Uploaded bucket manifest", {
+        manifestFileId: manifestFile.$id,
+        bucketId
+    });
+    return manifestFile.$id;
+}

package/dist/backups/operations/collectionBackup.d.ts

@@ -0,0 +1,30 @@
+import type { Storage, Databases } from "node-appwrite";
+import type { DatabaseAdapter } from "../../adapters/DatabaseAdapter.js";
+import type { AppwriteConfig } from "appwrite-utils";
+export interface CollectionBackupOptions {
+    trackingDatabaseId: string;
+    databaseId: string;
+    collectionIds: string[];
+    backupFormat?: 'json' | 'zip';
+    onProgress?: (message: string) => void;
+}
+export interface CollectionBackupResult {
+    backupId: string;
+    manifestFileId: string;
+    databaseId: string;
+    collections: Array<{
+        collectionId: string;
+        collectionName: string;
+        documentCount: number;
+        status: 'completed' | 'failed';
+        error?: string;
+    }>;
+    totalDocuments: number;
+    sizeBytes: number;
+    status: 'completed' | 'partial' | 'failed';
+    errors: string[];
+}
+/**
+ * Backup specific collections from a database
+ */
+export declare function backupCollections(config: AppwriteConfig, databases: Databases, storage: Storage, adapter: DatabaseAdapter, options: CollectionBackupOptions): Promise<CollectionBackupResult>;

package/dist/backups/operations/collectionBackup.js

@@ -0,0 +1,201 @@
+import { ID, Query } from "node-appwrite";
+import { InputFile } from "node-appwrite/file";
+import { ulid } from "ulidx";
+import { MessageFormatter } from "../../shared/messageFormatter.js";
+import { logger } from "../../shared/logging.js";
+import { tryAwaitWithRetry } from "appwrite-utils";
+import { splitIntoBatches } from "../../shared/migrationHelpers.js";
+import { retryFailedPromises } from "../../utils/retryFailedPromises.js";
+import { ProgressManager } from "../../shared/progressManager.js";
+import { createBackupZip } from "../../storage/backupCompression.js";
+import { recordCentralizedBackup, createCentralizedBackupTrackingTable } from "../tracking/centralizedTracking.js";
+/**
+ * Backup specific collections from a database
+ */
+export async function backupCollections(config, databases, storage, adapter, options) {
+    const startTime = Date.now();
+    const backupId = ulid();
+    const errors = [];
+    const collections = [];
+    let totalDocuments = 0;
+    let totalSizeBytes = 0;
+    try {
+        // Ensure tracking table exists
+        await createCentralizedBackupTrackingTable(adapter, options.trackingDatabaseId);
+        const backupBucketId = "appwrite-backups";
+        MessageFormatter.info(`Starting collection backup ${backupId}`, { prefix: "Backup" });
+        MessageFormatter.info(`Database: ${options.databaseId}, Collections: ${options.collectionIds.length}`, { prefix: "Backup" });
+        // Get database info
+        const db = await tryAwaitWithRetry(async () => await databases.get(options.databaseId));
+        const backupData = {
+            database: JSON.stringify(db),
+            collections: [],
+            documents: []
+        };
+        // Phase 1: Count documents for progress tracking
+        MessageFormatter.step(1, 3, "Analyzing collections");
+        let totalItems = options.collectionIds.length; // Start with collection count
+        for (const collectionId of options.collectionIds) {
+            try {
+                const documentCount = await tryAwaitWithRetry(async () => (await databases.listDocuments(options.databaseId, collectionId, [Query.limit(1)])).total);
+                totalDocuments += documentCount;
+                totalItems += documentCount;
+            }
+            catch (error) {
+                MessageFormatter.warning(`Could not count documents in collection ${collectionId}`);
+            }
+        }
+        const progress = ProgressManager.create(`backup-collections-${backupId}`, totalItems, {
+            title: `Backing up ${options.collectionIds.length} collections`,
+        });
+        // Phase 2: Backup selected collections
+        MessageFormatter.step(2, 3, `Processing ${options.collectionIds.length} collections and ${totalDocuments} documents`);
+        let processedDocuments = 0;
+        for (const collectionId of options.collectionIds) {
+            try {
+                if (options.onProgress) {
+                    options.onProgress(`Backing up collection: ${collectionId}`);
+                }
+                // Get collection metadata
+                const collection = await tryAwaitWithRetry(async () => await databases.getCollection(options.databaseId, collectionId));
+                backupData.collections.push(JSON.stringify(collection));
+                progress.increment(1, `Processing collection: ${collection.name}`);
+                // Backup all documents in this collection
+                let lastDocumentId = "";
+                let moreDocuments = true;
+                let collectionDocumentCount = 0;
+                while (moreDocuments) {
+                    const documentResponse = await tryAwaitWithRetry(async () => await databases.listDocuments(options.databaseId, collectionId, [
+                        Query.limit(500),
+                        ...(lastDocumentId ? [Query.cursorAfter(lastDocumentId)] : []),
+                    ]));
+                    collectionDocumentCount += documentResponse.documents.length;
+                    const documentPromises = documentResponse.documents.map(({ $id: documentId }) => databases.getDocument(options.databaseId, collectionId, documentId));
+                    const promiseBatches = splitIntoBatches(documentPromises);
+                    const documentsPulled = [];
+                    for (const batch of promiseBatches) {
+                        const successfulDocuments = await retryFailedPromises(batch);
+                        documentsPulled.push(...successfulDocuments);
+                        // Update progress for each batch
+                        progress.increment(successfulDocuments.length, `Processing ${collection.name}: ${processedDocuments + successfulDocuments.length}/${totalDocuments} documents`);
+                        processedDocuments += successfulDocuments.length;
+                    }
+                    backupData.documents.push({
+                        collectionId: collectionId,
+                        data: JSON.stringify(documentsPulled),
+                    });
+                    moreDocuments = documentResponse.documents.length === 500;
+                    if (moreDocuments) {
+                        lastDocumentId = documentResponse.documents[documentResponse.documents.length - 1].$id;
+                    }
+                }
+                collections.push({
+                    collectionId,
+                    collectionName: collection.name,
+                    documentCount: collectionDocumentCount,
+                    status: 'completed'
+                });
+                MessageFormatter.success(`Collection ${collection.name} backed up with ${MessageFormatter.formatNumber(collectionDocumentCount)} documents`);
+            }
+            catch (error) {
+                const errorMsg = `Failed to backup collection ${collectionId}: ${error instanceof Error ? error.message : String(error)}`;
+                errors.push(errorMsg);
+                logger.error(errorMsg);
+                collections.push({
+                    collectionId,
+                    collectionName: collectionId,
+                    documentCount: 0,
+                    status: 'failed',
+                    error: errorMsg
+                });
+            }
+        }
+        // Phase 3: Create backup file
+        MessageFormatter.step(3, 3, "Creating backup file");
+        let inputFile;
+        let fileName;
+        let backupSize;
+        if (options.backupFormat === 'zip') {
+            // Create compressed backup
+            const zipBuffer = await createBackupZip(backupData);
+            fileName = `${new Date().toISOString()}-${options.databaseId}-collections.zip`;
+            backupSize = zipBuffer.length;
+            inputFile = InputFile.fromBuffer(new Uint8Array(zipBuffer), fileName);
+        }
+        else {
+            // Use JSON format
+            const backupDataString = JSON.stringify(backupData, null, 2);
+            fileName = `${new Date().toISOString()}-${options.databaseId}-collections.json`;
+            backupSize = Buffer.byteLength(backupDataString, 'utf8');
+            inputFile = InputFile.fromPlainText(backupDataString, fileName);
+        }
+        const fileCreated = await storage.createFile(backupBucketId, ulid(), inputFile);
+        totalSizeBytes = backupSize;
+        // Create manifest
+        const manifestData = {
+            version: "1.0",
+            backupId,
+            databaseId: options.databaseId,
+            collectionIds: options.collectionIds,
+            collections: collections,
+            format: options.backupFormat || 'json',
+            createdAt: new Date().toISOString(),
+            totalDocuments: processedDocuments,
+            totalSizeBytes: backupSize
+        };
+        const manifestBuffer = Buffer.from(JSON.stringify(manifestData, null, 2), 'utf-8');
+        const manifestFile = await storage.createFile(backupBucketId, ID.unique(), InputFile.fromBuffer(new Uint8Array(manifestBuffer), `${backupId}-manifest.json`));
+        // Record in centralized tracking
+        await recordCentralizedBackup(adapter, options.trackingDatabaseId, {
+            backupType: 'database',
+            backupId: fileCreated.$id,
+            manifestFileId: manifestFile.$id,
+            format: options.backupFormat || 'json',
+            sizeBytes: backupSize,
+            databaseId: options.databaseId,
+            collections: backupData.collections.length,
+            documents: processedDocuments,
+            status: errors.length === 0 ? 'completed' : 'partial',
+            error: errors.length > 0 ? errors.join('; ') : undefined,
+            restorationStatus: 'not_restored'
+        });
+        progress.stop();
+        const duration = Date.now() - startTime;
+        const status = errors.length === 0 ? 'completed' :
+            collections.some(c => c.status === 'completed') ? 'partial' :
+                'failed';
+        MessageFormatter.success(`Collection backup ${status} in ${(duration / 1000).toFixed(2)}s`, { prefix: "Backup" });
+        MessageFormatter.operationSummary("Collection Backup", {
+            database: options.databaseId,
+            collections: backupData.collections.length,
+            documents: processedDocuments,
+            fileSize: MessageFormatter.formatBytes(backupSize),
+            backupFile: fileName,
+            bucket: backupBucketId,
+        }, duration);
+        return {
+            backupId,
+            manifestFileId: manifestFile.$id,
+            databaseId: options.databaseId,
+            collections,
+            totalDocuments: processedDocuments,
+            sizeBytes: totalSizeBytes,
+            status,
+            errors
+        };
+    }
+    catch (error) {
+        const errorMsg = `Collection backup failed: ${error instanceof Error ? error.message : String(error)}`;
+        MessageFormatter.error(errorMsg, error instanceof Error ? error : new Error(errorMsg), { prefix: "Backup" });
+        return {
+            backupId,
+            manifestFileId: '',
+            databaseId: options.databaseId,
+            collections: [],
+            totalDocuments: 0,
+            sizeBytes: 0,
+            status: 'failed',
+            errors: [errorMsg, ...errors]
+        };
+    }
+}

package/dist/backups/operations/comprehensiveBackup.d.ts

@@ -0,0 +1,25 @@
+import type { Storage, Databases } from "node-appwrite";
+import type { DatabaseAdapter } from "../../adapters/DatabaseAdapter.js";
+import type { DatabaseBackupReference, BucketBackupReference } from "../schemas/comprehensiveManifest.js";
+import type { AppwriteConfig } from "appwrite-utils";
+export interface ComprehensiveBackupOptions {
+    trackingDatabaseId: string;
+    backupFormat?: 'json' | 'zip';
+    skipDatabases?: boolean;
+    skipBuckets?: boolean;
+    parallelDownloads?: number;
+    onProgress?: (message: string) => void;
+}
+export interface ComprehensiveBackupResult {
+    backupId: string;
+    manifestFileId: string;
+    databaseBackups: DatabaseBackupReference[];
+    bucketBackups: BucketBackupReference[];
+    totalSizeBytes: number;
+    status: 'completed' | 'partial' | 'failed';
+    errors: string[];
+}
+/**
+ * Orchestrates comprehensive backup of ALL databases and ALL storage buckets
+ */
+export declare function comprehensiveBackup(config: AppwriteConfig, databases: Databases, storage: Storage, adapter: DatabaseAdapter, options: ComprehensiveBackupOptions): Promise<ComprehensiveBackupResult>;