appwrite-utils-cli 1.9.7 → 1.11.0

package/dist/backups/operations/bucketBackup.js

@@ -1,197 +0,0 @@
-import JSZip from "jszip";
-import { ID, Query } from "node-appwrite";
-import { InputFile } from "node-appwrite/file";
-import pLimit from "p-limit";
-import { MessageFormatter } from "../../shared/messageFormatter.js";
-import { logger } from "../../shared/logging.js";
-import { ulid } from "ulidx";
-/**
- * Downloads all files from a bucket in parallel and creates a ZIP backup
- */
-export async function backupBucket(storage, bucketId, backupBucketId, options = {}) {
-    const { compressionLevel = 6, parallelDownloads = 10, onProgress } = options;
-    const errors = [];
-    let totalSizeBytes = 0;
-    try {
-        // Step 1: Get bucket metadata
-        MessageFormatter.info(`Fetching bucket metadata for ${bucketId}`, { prefix: "Backup" });
-        const bucket = await storage.getBucket(bucketId);
-        // Step 2: List all files in bucket with pagination
-        MessageFormatter.info(`Listing all files in bucket ${bucketId}`, { prefix: "Backup" });
-        const allFiles = [];
-        let lastFileId;
-        while (true) {
-            const queries = [Query.limit(100)];
-            if (lastFileId) {
-                queries.push(Query.cursorAfter(lastFileId));
-            }
-            const filesPage = await storage.listFiles(bucketId, queries);
-            allFiles.push(...filesPage.files);
-            if (filesPage.files.length < 100)
-                break;
-            lastFileId = filesPage.files[filesPage.files.length - 1].$id;
-        }
-        MessageFormatter.info(`Found ${allFiles.length} files to backup`, { prefix: "Backup" });
-        if (allFiles.length === 0) {
-            // Empty bucket - create minimal backup
-            const manifest = {
-                version: "1.0",
-                bucketId: bucket.$id,
-                bucketName: bucket.name,
-                createdAt: new Date().toISOString(),
-                fileCount: 0,
-                totalSizeBytes: 0,
-                compression: bucket.compression === 'gzip' ? 'gzip' : 'none',
-                files: [],
-                bucketConfiguration: {
-                    $permissions: bucket.$permissions,
-                    fileSecurity: bucket.fileSecurity,
-                    enabled: bucket.enabled,
-                    maximumFileSize: bucket.maximumFileSize,
-                    allowedFileExtensions: bucket.allowedFileExtensions,
-                    compression: bucket.compression,
-                    encryption: bucket.encryption,
-                    antivirus: bucket.antivirus
-                }
-            };
-            const manifestFileId = await uploadManifest(storage, backupBucketId, bucketId, manifest);
-            return {
-                backupFileId: '',
-                manifestFileId,
-                fileCount: 0,
-                totalSizeBytes: 0,
-                zipSizeBytes: 0,
-                status: 'completed'
-            };
-        }
-        // Step 3: Download all files in parallel with concurrency limit
-        MessageFormatter.info(`Downloading ${allFiles.length} files in parallel (max ${parallelDownloads} concurrent)`, { prefix: "Backup" });
-        const limit = pLimit(parallelDownloads);
-        const downloadedFiles = new Map();
-        let successCount = 0;
-        let errorCount = 0;
-        const downloadTasks = allFiles.map((file, index) => limit(async () => {
-            try {
-                const fileBuffer = await storage.getFileDownload(bucketId, file.$id);
-                const buffer = Buffer.from(fileBuffer);
-                downloadedFiles.set(file.$id, { buffer, file });
-                successCount++;
-                totalSizeBytes += file.sizeOriginal || buffer.length;
-                if (onProgress) {
-                    onProgress(successCount + errorCount, allFiles.length, file.name);
-                }
-                logger.debug(`Downloaded file ${file.name}`, {
-                    fileId: file.$id,
-                    size: buffer.length
-                });
-            }
-            catch (error) {
-                errorCount++;
-                const errorMsg = `Failed to download file ${file.name} (${file.$id}): ${error instanceof Error ? error.message : String(error)}`;
-                errors.push(errorMsg);
-                logger.error(errorMsg);
-                if (onProgress) {
-                    onProgress(successCount + errorCount, allFiles.length, file.name);
-                }
-            }
-        }));
-        await Promise.all(downloadTasks);
-        if (successCount === 0) {
-            throw new Error(`Failed to download any files from bucket ${bucketId}`);
-        }
-        MessageFormatter.info(`Successfully downloaded ${successCount}/${allFiles.length} files`, { prefix: "Backup" });
-        // Step 4: Create ZIP archive with all files
-        MessageFormatter.info(`Creating ZIP archive for bucket ${bucketId}`, { prefix: "Backup" });
-        const zip = new JSZip();
-        for (const [fileId, { buffer, file }] of downloadedFiles.entries()) {
-            // Preserve file structure by using file name
-            const fileName = file.name || `file_${fileId}`;
-            zip.file(fileName, new Uint8Array(buffer));
-        }
-        // Generate ZIP buffer
-        const zipBuffer = await zip.generateAsync({
-            type: "nodebuffer",
-            compression: "DEFLATE",
-            compressionOptions: { level: compressionLevel }
-        });
-        const zipSizeBytes = zipBuffer.length;
-        MessageFormatter.info(`ZIP archive created: ${MessageFormatter.formatBytes(zipSizeBytes)}`, { prefix: "Backup" });
-        // Step 5: Create manifest
-        const fileMetadata = Array.from(downloadedFiles.values()).map(({ file }) => ({
-            $id: file.$id,
-            name: file.name,
-            size: file.sizeOriginal,
-            mimeType: file.mimeType,
-            $permissions: file.$permissions,
-            chunksCount: file.chunksTotal,
-            signature: file.signature,
-            $createdAt: file.$createdAt,
-            $updatedAt: file.$updatedAt
-        }));
-        const manifest = {
-            version: "1.0",
-            bucketId: bucket.$id,
-            bucketName: bucket.name,
-            createdAt: new Date().toISOString(),
-            fileCount: successCount,
-            totalSizeBytes,
-            compression: bucket.compression === 'gzip' ? 'gzip' : 'none',
-            files: fileMetadata,
-            bucketConfiguration: {
-                $permissions: bucket.$permissions,
-                fileSecurity: bucket.fileSecurity,
-                enabled: bucket.enabled,
-                maximumFileSize: bucket.maximumFileSize,
-                allowedFileExtensions: bucket.allowedFileExtensions,
-                compression: bucket.compression,
-                encryption: bucket.encryption,
-                antivirus: bucket.antivirus
-            }
-        };
-        // Step 6: Upload backup ZIP to backup bucket
-        MessageFormatter.info(`Uploading backup ZIP to bucket ${backupBucketId}`, { prefix: "Backup" });
-        const backupFileName = `${bucketId}.zip`;
-        const backupFile = await storage.createFile(backupBucketId, ID.unique(), InputFile.fromBuffer(new Uint8Array(zipBuffer), backupFileName));
-        // Step 7: Upload manifest JSON
-        const manifestFileId = await uploadManifest(storage, backupBucketId, bucketId, manifest);
-        const status = errorCount === 0 ? 'completed' :
-            successCount > 0 ? 'partial' :
-                'failed';
-        MessageFormatter.success(`Bucket backup ${status}: ${successCount}/${allFiles.length} files backed up`, { prefix: "Backup" });
-        return {
-            backupFileId: backupFile.$id,
-            manifestFileId,
-            fileCount: successCount,
-            totalSizeBytes,
-            zipSizeBytes,
-            status,
-            errors: errors.length > 0 ? errors : undefined
-        };
-    }
-    catch (error) {
-        const errorMsg = `Bucket backup failed: ${error instanceof Error ? error.message : String(error)}`;
-        MessageFormatter.error(errorMsg, error instanceof Error ? error : new Error(errorMsg), { prefix: "Backup" });
-        return {
-            backupFileId: '',
-            manifestFileId: '',
-            fileCount: 0,
-            totalSizeBytes: 0,
-            zipSizeBytes: 0,
-            status: 'failed',
-            errors: [errorMsg, ...errors]
-        };
-    }
-}
-/**
- * Uploads bucket manifest JSON to backup bucket
- */
-async function uploadManifest(storage, backupBucketId, bucketId, manifest) {
-    const manifestFileName = `${bucketId}.json`;
-    const manifestBuffer = Buffer.from(JSON.stringify(manifest, null, 2), 'utf-8');
-    const manifestFile = await storage.createFile(backupBucketId, ID.unique(), InputFile.fromBuffer(new Uint8Array(manifestBuffer), manifestFileName));
-    logger.info("Uploaded bucket manifest", {
-        manifestFileId: manifestFile.$id,
-        bucketId
-    });
-    return manifestFile.$id;
-}

package/dist/backups/operations/collectionBackup.d.ts

@@ -1,30 +0,0 @@
-import type { Storage, Databases } from "node-appwrite";
-import type { DatabaseAdapter } from "../../adapters/DatabaseAdapter.js";
-import type { AppwriteConfig } from "appwrite-utils";
-export interface CollectionBackupOptions {
-    trackingDatabaseId: string;
-    databaseId: string;
-    collectionIds: string[];
-    backupFormat?: 'json' | 'zip';
-    onProgress?: (message: string) => void;
-}
-export interface CollectionBackupResult {
-    backupId: string;
-    manifestFileId: string;
-    databaseId: string;
-    collections: Array<{
-        collectionId: string;
-        collectionName: string;
-        documentCount: number;
-        status: 'completed' | 'failed';
-        error?: string;
-    }>;
-    totalDocuments: number;
-    sizeBytes: number;
-    status: 'completed' | 'partial' | 'failed';
-    errors: string[];
-}
-/**
- * Backup specific collections from a database
- */
-export declare function backupCollections(config: AppwriteConfig, databases: Databases, storage: Storage, adapter: DatabaseAdapter, options: CollectionBackupOptions): Promise<CollectionBackupResult>;

package/dist/backups/operations/collectionBackup.js

@@ -1,201 +0,0 @@
-import { ID, Query } from "node-appwrite";
-import { InputFile } from "node-appwrite/file";
-import { ulid } from "ulidx";
-import { MessageFormatter } from "../../shared/messageFormatter.js";
-import { logger } from "../../shared/logging.js";
-import { tryAwaitWithRetry } from "appwrite-utils";
-import { splitIntoBatches } from "../../shared/migrationHelpers.js";
-import { retryFailedPromises } from "../../utils/retryFailedPromises.js";
-import { ProgressManager } from "../../shared/progressManager.js";
-import { createBackupZip } from "../../storage/backupCompression.js";
-import { recordCentralizedBackup, createCentralizedBackupTrackingTable } from "../tracking/centralizedTracking.js";
-/**
- * Backup specific collections from a database
- */
-export async function backupCollections(config, databases, storage, adapter, options) {
-    const startTime = Date.now();
-    const backupId = ulid();
-    const errors = [];
-    const collections = [];
-    let totalDocuments = 0;
-    let totalSizeBytes = 0;
-    try {
-        // Ensure tracking table exists
-        await createCentralizedBackupTrackingTable(adapter, options.trackingDatabaseId);
-        const backupBucketId = "appwrite-backups";
-        MessageFormatter.info(`Starting collection backup ${backupId}`, { prefix: "Backup" });
-        MessageFormatter.info(`Database: ${options.databaseId}, Collections: ${options.collectionIds.length}`, { prefix: "Backup" });
-        // Get database info
-        const db = await tryAwaitWithRetry(async () => await databases.get(options.databaseId));
-        const backupData = {
-            database: JSON.stringify(db),
-            collections: [],
-            documents: []
-        };
-        // Phase 1: Count documents for progress tracking
-        MessageFormatter.step(1, 3, "Analyzing collections");
-        let totalItems = options.collectionIds.length; // Start with collection count
-        for (const collectionId of options.collectionIds) {
-            try {
-                const documentCount = await tryAwaitWithRetry(async () => (await databases.listDocuments(options.databaseId, collectionId, [Query.limit(1)])).total);
-                totalDocuments += documentCount;
-                totalItems += documentCount;
-            }
-            catch (error) {
-                MessageFormatter.warning(`Could not count documents in collection ${collectionId}`);
-            }
-        }
-        const progress = ProgressManager.create(`backup-collections-${backupId}`, totalItems, {
-            title: `Backing up ${options.collectionIds.length} collections`,
-        });
-        // Phase 2: Backup selected collections
-        MessageFormatter.step(2, 3, `Processing ${options.collectionIds.length} collections and ${totalDocuments} documents`);
-        let processedDocuments = 0;
-        for (const collectionId of options.collectionIds) {
-            try {
-                if (options.onProgress) {
-                    options.onProgress(`Backing up collection: ${collectionId}`);
-                }
-                // Get collection metadata
-                const collection = await tryAwaitWithRetry(async () => await databases.getCollection(options.databaseId, collectionId));
-                backupData.collections.push(JSON.stringify(collection));
-                progress.increment(1, `Processing collection: ${collection.name}`);
-                // Backup all documents in this collection
-                let lastDocumentId = "";
-                let moreDocuments = true;
-                let collectionDocumentCount = 0;
-                while (moreDocuments) {
-                    const documentResponse = await tryAwaitWithRetry(async () => await databases.listDocuments(options.databaseId, collectionId, [
-                        Query.limit(500),
-                        ...(lastDocumentId ? [Query.cursorAfter(lastDocumentId)] : []),
-                    ]));
-                    collectionDocumentCount += documentResponse.documents.length;
-                    const documentPromises = documentResponse.documents.map(({ $id: documentId }) => databases.getDocument(options.databaseId, collectionId, documentId));
-                    const promiseBatches = splitIntoBatches(documentPromises);
-                    const documentsPulled = [];
-                    for (const batch of promiseBatches) {
-                        const successfulDocuments = await retryFailedPromises(batch);
-                        documentsPulled.push(...successfulDocuments);
-                        // Update progress for each batch
-                        progress.increment(successfulDocuments.length, `Processing ${collection.name}: ${processedDocuments + successfulDocuments.length}/${totalDocuments} documents`);
-                        processedDocuments += successfulDocuments.length;
-                    }
-                    backupData.documents.push({
-                        collectionId: collectionId,
-                        data: JSON.stringify(documentsPulled),
-                    });
-                    moreDocuments = documentResponse.documents.length === 500;
-                    if (moreDocuments) {
-                        lastDocumentId = documentResponse.documents[documentResponse.documents.length - 1].$id;
-                    }
-                }
-                collections.push({
-                    collectionId,
-                    collectionName: collection.name,
-                    documentCount: collectionDocumentCount,
-                    status: 'completed'
-                });
-                MessageFormatter.success(`Collection ${collection.name} backed up with ${MessageFormatter.formatNumber(collectionDocumentCount)} documents`);
-            }
-            catch (error) {
-                const errorMsg = `Failed to backup collection ${collectionId}: ${error instanceof Error ? error.message : String(error)}`;
-                errors.push(errorMsg);
-                logger.error(errorMsg);
-                collections.push({
-                    collectionId,
-                    collectionName: collectionId,
-                    documentCount: 0,
-                    status: 'failed',
-                    error: errorMsg
-                });
-            }
-        }
-        // Phase 3: Create backup file
-        MessageFormatter.step(3, 3, "Creating backup file");
-        let inputFile;
-        let fileName;
-        let backupSize;
-        if (options.backupFormat === 'zip') {
-            // Create compressed backup
-            const zipBuffer = await createBackupZip(backupData);
-            fileName = `${new Date().toISOString()}-${options.databaseId}-collections.zip`;
-            backupSize = zipBuffer.length;
-            inputFile = InputFile.fromBuffer(new Uint8Array(zipBuffer), fileName);
-        }
-        else {
-            // Use JSON format
-            const backupDataString = JSON.stringify(backupData, null, 2);
-            fileName = `${new Date().toISOString()}-${options.databaseId}-collections.json`;
-            backupSize = Buffer.byteLength(backupDataString, 'utf8');
-            inputFile = InputFile.fromPlainText(backupDataString, fileName);
-        }
-        const fileCreated = await storage.createFile(backupBucketId, ulid(), inputFile);
-        totalSizeBytes = backupSize;
-        // Create manifest
-        const manifestData = {
-            version: "1.0",
-            backupId,
-            databaseId: options.databaseId,
-            collectionIds: options.collectionIds,
-            collections: collections,
-            format: options.backupFormat || 'json',
-            createdAt: new Date().toISOString(),
-            totalDocuments: processedDocuments,
-            totalSizeBytes: backupSize
-        };
-        const manifestBuffer = Buffer.from(JSON.stringify(manifestData, null, 2), 'utf-8');
-        const manifestFile = await storage.createFile(backupBucketId, ID.unique(), InputFile.fromBuffer(new Uint8Array(manifestBuffer), `${backupId}-manifest.json`));
-        // Record in centralized tracking
-        await recordCentralizedBackup(adapter, options.trackingDatabaseId, {
-            backupType: 'database',
-            backupId: fileCreated.$id,
-            manifestFileId: manifestFile.$id,
-            format: options.backupFormat || 'json',
-            sizeBytes: backupSize,
-            databaseId: options.databaseId,
-            collections: backupData.collections.length,
-            documents: processedDocuments,
-            status: errors.length === 0 ? 'completed' : 'partial',
-            error: errors.length > 0 ? errors.join('; ') : undefined,
-            restorationStatus: 'not_restored'
-        });
-        progress.stop();
-        const duration = Date.now() - startTime;
-        const status = errors.length === 0 ? 'completed' :
-            collections.some(c => c.status === 'completed') ? 'partial' :
-                'failed';
-        MessageFormatter.success(`Collection backup ${status} in ${(duration / 1000).toFixed(2)}s`, { prefix: "Backup" });
-        MessageFormatter.operationSummary("Collection Backup", {
-            database: options.databaseId,
-            collections: backupData.collections.length,
-            documents: processedDocuments,
-            fileSize: MessageFormatter.formatBytes(backupSize),
-            backupFile: fileName,
-            bucket: backupBucketId,
-        }, duration);
-        return {
-            backupId,
-            manifestFileId: manifestFile.$id,
-            databaseId: options.databaseId,
-            collections,
-            totalDocuments: processedDocuments,
-            sizeBytes: totalSizeBytes,
-            status,
-            errors
-        };
-    }
-    catch (error) {
-        const errorMsg = `Collection backup failed: ${error instanceof Error ? error.message : String(error)}`;
-        MessageFormatter.error(errorMsg, error instanceof Error ? error : new Error(errorMsg), { prefix: "Backup" });
-        return {
-            backupId,
-            manifestFileId: '',
-            databaseId: options.databaseId,
-            collections: [],
-            totalDocuments: 0,
-            sizeBytes: 0,
-            status: 'failed',
-            errors: [errorMsg, ...errors]
-        };
-    }
-}

package/dist/backups/operations/comprehensiveBackup.d.ts

@@ -1,25 +0,0 @@
-import type { Storage, Databases } from "node-appwrite";
-import type { DatabaseAdapter } from "../../adapters/DatabaseAdapter.js";
-import type { DatabaseBackupReference, BucketBackupReference } from "../schemas/comprehensiveManifest.js";
-import type { AppwriteConfig } from "appwrite-utils";
-export interface ComprehensiveBackupOptions {
-    trackingDatabaseId: string;
-    backupFormat?: 'json' | 'zip';
-    skipDatabases?: boolean;
-    skipBuckets?: boolean;
-    parallelDownloads?: number;
-    onProgress?: (message: string) => void;
-}
-export interface ComprehensiveBackupResult {
-    backupId: string;
-    manifestFileId: string;
-    databaseBackups: DatabaseBackupReference[];
-    bucketBackups: BucketBackupReference[];
-    totalSizeBytes: number;
-    status: 'completed' | 'partial' | 'failed';
-    errors: string[];
-}
-/**
- * Orchestrates comprehensive backup of ALL databases and ALL storage buckets
- */
-export declare function comprehensiveBackup(config: AppwriteConfig, databases: Databases, storage: Storage, adapter: DatabaseAdapter, options: ComprehensiveBackupOptions): Promise<ComprehensiveBackupResult>;