appwrite-utils-cli 1.11.0 → 1.12.0

package/src/backups/operations/bucketBackup.ts

@@ -1,277 +0,0 @@
-import type { Storage, Models } from "node-appwrite";
-import JSZip from "jszip";
-import { ID, Query } from "node-appwrite";
-import { InputFile } from "node-appwrite/file";
-import pLimit from "p-limit";
-import { MessageFormatter } from 'appwrite-utils-helpers';
-import { logger } from 'appwrite-utils-helpers';
-import type { BucketManifest, BucketFileMetadata } from "../schemas/bucketManifest.js";
-import { ulid } from "ulidx";
-
-export interface BucketBackupOptions {
-  compressionLevel?: number; // 0-9, default 6
-  parallelDownloads?: number; // Number of concurrent downloads, default 10
-  onProgress?: (current: number, total: number, fileName: string) => void;
-}
-
-export interface BucketBackupResult {
-  backupFileId: string;
-  manifestFileId: string;
-  fileCount: number;
-  totalSizeBytes: number;
-  zipSizeBytes: number;
-  status: 'completed' | 'partial' | 'failed';
-  errors?: string[];
-}
-
-/**
- * Downloads all files from a bucket in parallel and creates a ZIP backup
- */
-export async function backupBucket(
-  storage: Storage,
-  bucketId: string,
-  backupBucketId: string,
-  options: BucketBackupOptions = {}
-): Promise<BucketBackupResult> {
-  const {
-    compressionLevel = 6,
-    parallelDownloads = 10,
-    onProgress
-  } = options;
-
-  const errors: string[] = [];
-  let totalSizeBytes = 0;
-
-  try {
-    // Step 1: Get bucket metadata
-    MessageFormatter.info(`Fetching bucket metadata for ${bucketId}`, { prefix: "Backup" });
-    const bucket = await storage.getBucket(bucketId);
-
-    // Step 2: List all files in bucket with pagination
-    MessageFormatter.info(`Listing all files in bucket ${bucketId}`, { prefix: "Backup" });
-    const allFiles: Models.File[] = [];
-    let lastFileId: string | undefined;
-
-    while (true) {
-      const queries = [Query.limit(100)];
-      if (lastFileId) {
-        queries.push(Query.cursorAfter(lastFileId));
-      }
-
-      const filesPage = await storage.listFiles(bucketId, queries);
-      allFiles.push(...filesPage.files);
-
-      if (filesPage.files.length < 100) break;
-      lastFileId = filesPage.files[filesPage.files.length - 1].$id;
-    }
-
-    MessageFormatter.info(`Found ${allFiles.length} files to backup`, { prefix: "Backup" });
-
-    if (allFiles.length === 0) {
-      // Empty bucket - create minimal backup
-      const manifest: BucketManifest = {
-        version: "1.0",
-        bucketId: bucket.$id,
-        bucketName: bucket.name,
-        createdAt: new Date().toISOString(),
-        fileCount: 0,
-        totalSizeBytes: 0,
-        compression: bucket.compression === 'gzip' ? 'gzip' : 'none',
-        files: [],
-        bucketConfiguration: {
-          $permissions: bucket.$permissions,
-          fileSecurity: bucket.fileSecurity,
-          enabled: bucket.enabled,
-          maximumFileSize: bucket.maximumFileSize,
-          allowedFileExtensions: bucket.allowedFileExtensions,
-          compression: bucket.compression,
-          encryption: bucket.encryption,
-          antivirus: bucket.antivirus
-        }
-      };
-
-      const manifestFileId = await uploadManifest(storage, backupBucketId, bucketId, manifest);
-
-      return {
-        backupFileId: '',
-        manifestFileId,
-        fileCount: 0,
-        totalSizeBytes: 0,
-        zipSizeBytes: 0,
-        status: 'completed'
-      };
-    }
-
-    // Step 3: Download all files in parallel with concurrency limit
-    MessageFormatter.info(`Downloading ${allFiles.length} files in parallel (max ${parallelDownloads} concurrent)`, { prefix: "Backup" });
-
-    const limit = pLimit(parallelDownloads);
-    const downloadedFiles: Map<string, { buffer: Buffer; file: Models.File }> = new Map();
-    let successCount = 0;
-    let errorCount = 0;
-
-    const downloadTasks = allFiles.map((file, index) =>
-      limit(async () => {
-        try {
-          const fileBuffer = await storage.getFileDownload(bucketId, file.$id);
-          const buffer = Buffer.from(fileBuffer as ArrayBuffer);
-
-          downloadedFiles.set(file.$id, { buffer, file });
-          successCount++;
-          totalSizeBytes += file.sizeOriginal || buffer.length;
-
-          if (onProgress) {
-            onProgress(successCount + errorCount, allFiles.length, file.name);
-          }
-
-          logger.debug(`Downloaded file ${file.name}`, {
-            fileId: file.$id,
-            size: buffer.length
-          });
-        } catch (error) {
-          errorCount++;
-          const errorMsg = `Failed to download file ${file.name} (${file.$id}): ${error instanceof Error ? error.message : String(error)}`;
-          errors.push(errorMsg);
-          logger.error(errorMsg);
-
-          if (onProgress) {
-            onProgress(successCount + errorCount, allFiles.length, file.name);
-          }
-        }
-      })
-    );
-
-    await Promise.all(downloadTasks);
-
-    if (successCount === 0) {
-      throw new Error(`Failed to download any files from bucket ${bucketId}`);
-    }
-
-    MessageFormatter.info(`Successfully downloaded ${successCount}/${allFiles.length} files`, { prefix: "Backup" });
-
-    // Step 4: Create ZIP archive with all files
-    MessageFormatter.info(`Creating ZIP archive for bucket ${bucketId}`, { prefix: "Backup" });
-    const zip = new JSZip();
-
-    for (const [fileId, { buffer, file }] of downloadedFiles.entries()) {
-      // Preserve file structure by using file name
-      const fileName = file.name || `file_${fileId}`;
-      zip.file(fileName, new Uint8Array(buffer));
-    }
-
-    // Generate ZIP buffer
-    const zipBuffer = await zip.generateAsync({
-      type: "nodebuffer",
-      compression: "DEFLATE",
-      compressionOptions: { level: compressionLevel }
-    });
-
-    const zipSizeBytes = zipBuffer.length;
-    MessageFormatter.info(`ZIP archive created: ${MessageFormatter.formatBytes(zipSizeBytes)}`, { prefix: "Backup" });
-
-    // Step 5: Create manifest
-    const fileMetadata: BucketFileMetadata[] = Array.from(downloadedFiles.values()).map(({ file }) => ({
-      $id: file.$id,
-      name: file.name,
-      size: file.sizeOriginal,
-      mimeType: file.mimeType,
-      $permissions: file.$permissions,
-      chunksCount: file.chunksTotal,
-      signature: file.signature,
-      $createdAt: file.$createdAt,
-      $updatedAt: file.$updatedAt
-    }));
-
-    const manifest: BucketManifest = {
-      version: "1.0",
-      bucketId: bucket.$id,
-      bucketName: bucket.name,
-      createdAt: new Date().toISOString(),
-      fileCount: successCount,
-      totalSizeBytes,
-      compression: bucket.compression === 'gzip' ? 'gzip' : 'none',
-      files: fileMetadata,
-      bucketConfiguration: {
-        $permissions: bucket.$permissions,
-        fileSecurity: bucket.fileSecurity,
-        enabled: bucket.enabled,
-        maximumFileSize: bucket.maximumFileSize,
-        allowedFileExtensions: bucket.allowedFileExtensions,
-        compression: bucket.compression,
-        encryption: bucket.encryption,
-        antivirus: bucket.antivirus
-      }
-    };
-
-    // Step 6: Upload backup ZIP to backup bucket
-    MessageFormatter.info(`Uploading backup ZIP to bucket ${backupBucketId}`, { prefix: "Backup" });
-    const backupFileName = `${bucketId}.zip`;
-    const backupFile = await storage.createFile(
-      backupBucketId,
-      ID.unique(),
-      InputFile.fromBuffer(new Uint8Array(zipBuffer), backupFileName)
-    );
-
-    // Step 7: Upload manifest JSON
-    const manifestFileId = await uploadManifest(storage, backupBucketId, bucketId, manifest);
-
-    const status: 'completed' | 'partial' | 'failed' =
-      errorCount === 0 ? 'completed' :
-      successCount > 0 ? 'partial' :
-      'failed';
-
-    MessageFormatter.success(
-      `Bucket backup ${status}: ${successCount}/${allFiles.length} files backed up`,
-      { prefix: "Backup" }
-    );
-
-    return {
-      backupFileId: backupFile.$id,
-      manifestFileId,
-      fileCount: successCount,
-      totalSizeBytes,
-      zipSizeBytes,
-      status,
-      errors: errors.length > 0 ? errors : undefined
-    };
-  } catch (error) {
-    const errorMsg = `Bucket backup failed: ${error instanceof Error ? error.message : String(error)}`;
-    MessageFormatter.error(errorMsg, error instanceof Error ? error : new Error(errorMsg), { prefix: "Backup" });
-
-    return {
-      backupFileId: '',
-      manifestFileId: '',
-      fileCount: 0,
-      totalSizeBytes: 0,
-      zipSizeBytes: 0,
-      status: 'failed',
-      errors: [errorMsg, ...errors]
-    };
-  }
-}
-
-/**
- * Uploads bucket manifest JSON to backup bucket
- */
-async function uploadManifest(
-  storage: Storage,
-  backupBucketId: string,
-  bucketId: string,
-  manifest: BucketManifest
-): Promise<string> {
-  const manifestFileName = `${bucketId}.json`;
-  const manifestBuffer = Buffer.from(JSON.stringify(manifest, null, 2), 'utf-8');
-
-  const manifestFile = await storage.createFile(
-    backupBucketId,
-    ID.unique(),
-    InputFile.fromBuffer(new Uint8Array(manifestBuffer), manifestFileName)
-  );
-
-  logger.info("Uploaded bucket manifest", {
-    manifestFileId: manifestFile.$id,
-    bucketId
-  });
-
-  return manifestFile.$id;
-}

package/src/backups/operations/collectionBackup.ts

@@ -1,310 +0,0 @@
-import type { Storage, Databases, Models } from "node-appwrite";
-import { ID, Query } from "node-appwrite";
-import { InputFile } from "node-appwrite/file";
-import { ulid } from "ulidx";
-import { MessageFormatter } from 'appwrite-utils-helpers';
-import { logger } from 'appwrite-utils-helpers';
-import type { DatabaseAdapter } from 'appwrite-utils-helpers';
-import { tryAwaitWithRetry } from "appwrite-utils";
-import { splitIntoBatches } from "../../shared/migrationHelpers.js";
-import { retryFailedPromises } from "appwrite-utils-helpers";
-import { ProgressManager } from "../../shared/progressManager.js";
-import { createBackupZip } from "../../storage/backupCompression.js";
-import {
-  recordCentralizedBackup,
-  createCentralizedBackupTrackingTable
-} from "../tracking/centralizedTracking.js";
-import type { AppwriteConfig } from "appwrite-utils";
-
-export interface CollectionBackupOptions {
-  trackingDatabaseId: string;
-  databaseId: string;
-  collectionIds: string[];
-  backupFormat?: 'json' | 'zip';
-  onProgress?: (message: string) => void;
-}
-
-export interface CollectionBackupResult {
-  backupId: string;
-  manifestFileId: string;
-  databaseId: string;
-  collections: Array<{
-    collectionId: string;
-    collectionName: string;
-    documentCount: number;
-    status: 'completed' | 'failed';
-    error?: string;
-  }>;
-  totalDocuments: number;
-  sizeBytes: number;
-  status: 'completed' | 'partial' | 'failed';
-  errors: string[];
-}
-
-interface BackupData {
-  database: string;
-  collections: string[];
-  documents: Array<{
-    collectionId: string;
-    data: string;
-  }>;
-}
-
-/**
- * Backup specific collections from a database
- */
-export async function backupCollections(
-  config: AppwriteConfig,
-  databases: Databases,
-  storage: Storage,
-  adapter: DatabaseAdapter,
-  options: CollectionBackupOptions
-): Promise<CollectionBackupResult> {
-  const startTime = Date.now();
-  const backupId = ulid();
-  const errors: string[] = [];
-  const collections: CollectionBackupResult['collections'] = [];
-  let totalDocuments = 0;
-  let totalSizeBytes = 0;
-
-  try {
-    // Ensure tracking table exists
-    await createCentralizedBackupTrackingTable(adapter, options.trackingDatabaseId);
-
-    const backupBucketId = "appwrite-backups";
-    MessageFormatter.info(`Starting collection backup ${backupId}`, { prefix: "Backup" });
-    MessageFormatter.info(`Database: ${options.databaseId}, Collections: ${options.collectionIds.length}`, { prefix: "Backup" });
-
-    // Get database info
-    const db = await tryAwaitWithRetry(
-      async () => await databases.get(options.databaseId)
-    );
-
-    const backupData: BackupData = {
-      database: JSON.stringify(db),
-      collections: [],
-      documents: []
-    };
-
-    // Phase 1: Count documents for progress tracking
-    MessageFormatter.step(1, 3, "Analyzing collections");
-    let totalItems = options.collectionIds.length; // Start with collection count
-
-    for (const collectionId of options.collectionIds) {
-      try {
-        const documentCount = await tryAwaitWithRetry(
-          async () => (await databases.listDocuments(options.databaseId, collectionId, [Query.limit(1)])).total
-        );
-        totalDocuments += documentCount;
-        totalItems += documentCount;
-      } catch (error) {
-        MessageFormatter.warning(`Could not count documents in collection ${collectionId}`);
-      }
-    }
-
-    const progress = ProgressManager.create(`backup-collections-${backupId}`, totalItems, {
-      title: `Backing up ${options.collectionIds.length} collections`,
-    });
-
-    // Phase 2: Backup selected collections
-    MessageFormatter.step(2, 3, `Processing ${options.collectionIds.length} collections and ${totalDocuments} documents`);
-
-    let processedDocuments = 0;
-
-    for (const collectionId of options.collectionIds) {
-      try {
-        if (options.onProgress) {
-          options.onProgress(`Backing up collection: ${collectionId}`);
-        }
-
-        // Get collection metadata
-        const collection = await tryAwaitWithRetry(
-          async () => await databases.getCollection(options.databaseId, collectionId)
-        );
-
-        backupData.collections.push(JSON.stringify(collection));
-        progress.increment(1, `Processing collection: ${collection.name}`);
-
-        // Backup all documents in this collection
-        let lastDocumentId = "";
-        let moreDocuments = true;
-        let collectionDocumentCount = 0;
-
-        while (moreDocuments) {
-          const documentResponse = await tryAwaitWithRetry(
-            async () =>
-              await databases.listDocuments(options.databaseId, collectionId, [
-                Query.limit(500),
-                ...(lastDocumentId ? [Query.cursorAfter(lastDocumentId)] : []),
-              ])
-          );
-
-          collectionDocumentCount += documentResponse.documents.length;
-
-          const documentPromises = documentResponse.documents.map(
-            ({ $id: documentId }) =>
-              databases.getDocument(options.databaseId, collectionId, documentId)
-          );
-
-          const promiseBatches = splitIntoBatches(documentPromises);
-          const documentsPulled = [];
-
-          for (const batch of promiseBatches) {
-            const successfulDocuments = await retryFailedPromises(batch);
-            documentsPulled.push(...successfulDocuments);
-
-            // Update progress for each batch
-            progress.increment(successfulDocuments.length,
-              `Processing ${collection.name}: ${processedDocuments + successfulDocuments.length}/${totalDocuments} documents`
-            );
-            processedDocuments += successfulDocuments.length;
-          }
-
-          backupData.documents.push({
-            collectionId: collectionId,
-            data: JSON.stringify(documentsPulled),
-          });
-
-          moreDocuments = documentResponse.documents.length === 500;
-          if (moreDocuments) {
-            lastDocumentId = documentResponse.documents[documentResponse.documents.length - 1].$id;
-          }
-        }
-
-        collections.push({
-          collectionId,
-          collectionName: collection.name,
-          documentCount: collectionDocumentCount,
-          status: 'completed'
-        });
-
-        MessageFormatter.success(
-          `Collection ${collection.name} backed up with ${MessageFormatter.formatNumber(collectionDocumentCount)} documents`
-        );
-      } catch (error) {
-        const errorMsg = `Failed to backup collection ${collectionId}: ${error instanceof Error ? error.message : String(error)}`;
-        errors.push(errorMsg);
-        logger.error(errorMsg);
-
-        collections.push({
-          collectionId,
-          collectionName: collectionId,
-          documentCount: 0,
-          status: 'failed',
-          error: errorMsg
-        });
-      }
-    }
-
-    // Phase 3: Create backup file
-    MessageFormatter.step(3, 3, "Creating backup file");
-
-    let inputFile: any;
-    let fileName: string;
-    let backupSize: number;
-
-    if (options.backupFormat === 'zip') {
-      // Create compressed backup
-      const zipBuffer = await createBackupZip(backupData);
-      fileName = `${new Date().toISOString()}-${options.databaseId}-collections.zip`;
-      backupSize = zipBuffer.length;
-      inputFile = InputFile.fromBuffer(new Uint8Array(zipBuffer), fileName);
-    } else {
-      // Use JSON format
-      const backupDataString = JSON.stringify(backupData, null, 2);
-      fileName = `${new Date().toISOString()}-${options.databaseId}-collections.json`;
-      backupSize = Buffer.byteLength(backupDataString, 'utf8');
-      inputFile = InputFile.fromPlainText(backupDataString, fileName);
-    }
-
-    const fileCreated = await storage.createFile(
-      backupBucketId,
-      ulid(),
-      inputFile
-    );
-
-    totalSizeBytes = backupSize;
-
-    // Create manifest
-    const manifestData = {
-      version: "1.0",
-      backupId,
-      databaseId: options.databaseId,
-      collectionIds: options.collectionIds,
-      collections: collections,
-      format: options.backupFormat || 'json',
-      createdAt: new Date().toISOString(),
-      totalDocuments: processedDocuments,
-      totalSizeBytes: backupSize
-    };
-
-    const manifestBuffer = Buffer.from(JSON.stringify(manifestData, null, 2), 'utf-8');
-    const manifestFile = await storage.createFile(
-      backupBucketId,
-      ID.unique(),
-      InputFile.fromBuffer(new Uint8Array(manifestBuffer), `${backupId}-manifest.json`)
-    );
-
-    // Record in centralized tracking
-    await recordCentralizedBackup(adapter, options.trackingDatabaseId, {
-      backupType: 'database',
-      backupId: fileCreated.$id,
-      manifestFileId: manifestFile.$id,
-      format: options.backupFormat || 'json',
-      sizeBytes: backupSize,
-      databaseId: options.databaseId,
-      collections: backupData.collections.length,
-      documents: processedDocuments,
-      status: errors.length === 0 ? 'completed' : 'partial',
-      error: errors.length > 0 ? errors.join('; ') : undefined,
-      restorationStatus: 'not_restored'
-    });
-
-    progress.stop();
-
-    const duration = Date.now() - startTime;
-    const status: 'completed' | 'partial' | 'failed' =
-      errors.length === 0 ? 'completed' :
-      collections.some(c => c.status === 'completed') ? 'partial' :
-      'failed';
-
-    MessageFormatter.success(
-      `Collection backup ${status} in ${(duration / 1000).toFixed(2)}s`,
-      { prefix: "Backup" }
-    );
-
-    MessageFormatter.operationSummary("Collection Backup", {
-      database: options.databaseId,
-      collections: backupData.collections.length,
-      documents: processedDocuments,
-      fileSize: MessageFormatter.formatBytes(backupSize),
-      backupFile: fileName,
-      bucket: backupBucketId,
-    }, duration);
-
-    return {
-      backupId,
-      manifestFileId: manifestFile.$id,
-      databaseId: options.databaseId,
-      collections,
-      totalDocuments: processedDocuments,
-      sizeBytes: totalSizeBytes,
-      status,
-      errors
-    };
-  } catch (error) {
-    const errorMsg = `Collection backup failed: ${error instanceof Error ? error.message : String(error)}`;
-    MessageFormatter.error(errorMsg, error instanceof Error ? error : new Error(errorMsg), { prefix: "Backup" });
-
-    return {
-      backupId,
-      manifestFileId: '',
-      databaseId: options.databaseId,
-      collections: [],
-      totalDocuments: 0,
-      sizeBytes: 0,
-      status: 'failed',
-      errors: [errorMsg, ...errors]
-    };
-  }
-}