@njdamstra/appwrite-utils-cli 1.8.9

package/src/backups/operations/comprehensiveBackup.ts

@@ -0,0 +1,342 @@
+import type { Storage, Databases, Models } from "node-appwrite";
+import { ID } from "node-appwrite";
+import { InputFile } from "node-appwrite/file";
+import { ulid } from "ulidx";
+import { MessageFormatter } from "../../shared/messageFormatter.js";
+import { logger } from "../../shared/logging.js";
+import type { DatabaseAdapter } from "../../adapters/DatabaseAdapter.js";
+import { backupDatabase } from "../../storage/methods.js";
+import { backupBucket } from "./bucketBackup.js";
+import {
+  recordCentralizedBackup,
+  createCentralizedBackupTrackingTable
+} from "../tracking/centralizedTracking.js";
+import type {
+  ComprehensiveManifest,
+  DatabaseBackupReference,
+  BucketBackupReference
+} from "../schemas/comprehensiveManifest.js";
+import type { AppwriteConfig } from "@njdamstra/appwrite-utils";
+import { fetchAllDatabases } from "../../databases/methods.js";
+
+export interface ComprehensiveBackupOptions {
+  trackingDatabaseId: string; // Database to store backup tracking
+  backupFormat?: 'json' | 'zip';
+  skipDatabases?: boolean;
+  skipBuckets?: boolean;
+  parallelDownloads?: number;
+  onProgress?: (message: string) => void;
+}
+
+export interface ComprehensiveBackupResult {
+  backupId: string;
+  manifestFileId: string;
+  databaseBackups: DatabaseBackupReference[];
+  bucketBackups: BucketBackupReference[];
+  totalSizeBytes: number;
+  status: 'completed' | 'partial' | 'failed';
+  errors: string[];
+}
+
+/**
+ * Orchestrates comprehensive backup of ALL databases and ALL storage buckets
+ */
+export async function comprehensiveBackup(
+  config: AppwriteConfig,
+  databases: Databases,
+  storage: Storage,
+  adapter: DatabaseAdapter,
+  options: ComprehensiveBackupOptions
+): Promise<ComprehensiveBackupResult> {
+  const startTime = Date.now();
+  const backupId = ulid();
+  const errors: string[] = [];
+  const databaseBackups: DatabaseBackupReference[] = [];
+  const bucketBackups: BucketBackupReference[] = [];
+  let totalSizeBytes = 0;
+
+  try {
+    // Ensure tracking table exists
+    await createCentralizedBackupTrackingTable(adapter, options.trackingDatabaseId);
+
+    // Initialize backup bucket
+    const backupBucketId = "appwrite-backups";
+    MessageFormatter.info(`Starting comprehensive backup ${backupId}`, { prefix: "Backup" });
+
+    // Phase 1: Backup ALL databases
+    if (!options.skipDatabases) {
+      MessageFormatter.info("Phase 1: Backing up ALL databases", { prefix: "Backup" });
+
+      const allDatabases = await fetchAllDatabases(databases);
+
+      // Validate each database exists before attempting backup
+      const validDatabases: Models.Database[] = [];
+      const skippedDatabases: string[] = [];
+
+      MessageFormatter.info(`Validating ${allDatabases.length} databases...`, { prefix: "Backup" });
+
+      for (const db of allDatabases) {
+        try {
+          await databases.get(db.$id); // Validate existence
+          validDatabases.push(db);
+        } catch (error) {
+          skippedDatabases.push(`${db.name} (${db.$id})`);
+          MessageFormatter.warning(
+            `Database ${db.name} not found - skipping`,
+            { prefix: "Backup" }
+          );
+          logger.warn('Database validation failed', {
+            databaseId: db.$id,
+            databaseName: db.name,
+            error: error instanceof Error ? error.message : String(error)
+          });
+        }
+      }
+
+      if (skippedDatabases.length > 0) {
+        MessageFormatter.info(
+          `Skipped ${skippedDatabases.length} invalid databases: ${skippedDatabases.join(', ')}`,
+          { prefix: "Backup" }
+        );
+      }
+
+      MessageFormatter.info(`Found ${validDatabases.length} valid databases to backup`, { prefix: "Backup" });
+
+      for (const db of validDatabases) {
+        try {
+          if (options.onProgress) {
+            options.onProgress(`Backing up database: ${db.name}`);
+          }
+
+          MessageFormatter.info(`Backing up database: ${db.name} (${db.$id})`, { prefix: "Backup" });
+
+          // Use existing backupDatabase function
+          const dbBackupResult = await backupDatabase(
+            config,
+            databases,
+            db.$id,
+            storage,
+            options.backupFormat || 'zip'
+          );
+
+          // Create database manifest with complete data
+          const manifestData = {
+            version: "1.0",
+            databaseId: dbBackupResult.databaseId,
+            databaseName: dbBackupResult.databaseName,
+            format: dbBackupResult.format,
+            collectionCount: dbBackupResult.collectionCount,
+            documentCount: dbBackupResult.documentCount,
+            backupFileId: dbBackupResult.backupFileId,
+            createdAt: new Date().toISOString()
+          };
+
+          const manifestBuffer = Buffer.from(JSON.stringify(manifestData, null, 2), 'utf-8');
+          const manifestFile = await storage.createFile(
+            backupBucketId,
+            ID.unique(),
+            InputFile.fromBuffer(new Uint8Array(manifestBuffer), `${db.$id}-manifest.json`)
+          );
+
+          databaseBackups.push({
+            databaseId: dbBackupResult.databaseId,
+            databaseName: dbBackupResult.databaseName,
+            backupFileId: dbBackupResult.backupFileId,
+            manifestFileId: manifestFile.$id,
+            collectionCount: dbBackupResult.collectionCount,
+            documentCount: dbBackupResult.documentCount,
+            sizeBytes: dbBackupResult.backupSizeBytes,
+            status: 'completed'
+          });
+
+          totalSizeBytes += dbBackupResult.backupSizeBytes;
+
+          // Record individual database backup in tracking
+          await recordCentralizedBackup(adapter, options.trackingDatabaseId, {
+            backupType: 'database',
+            backupId: dbBackupResult.backupFileId,
+            manifestFileId: manifestFile.$id,
+            format: dbBackupResult.format,
+            sizeBytes: dbBackupResult.backupSizeBytes,
+            databaseId: dbBackupResult.databaseId,
+            collections: dbBackupResult.collectionCount,
+            documents: dbBackupResult.documentCount,
+            status: 'completed',
+            restorationStatus: 'not_restored'
+          });
+
+          MessageFormatter.success(`Database ${db.name} backed up successfully`, { prefix: "Backup" });
+        } catch (error) {
+          const errorMsg = `Failed to backup database ${db.name}: ${error instanceof Error ? error.message : String(error)}`;
+          errors.push(errorMsg);
+          logger.error(errorMsg);
+
+          databaseBackups.push({
+            databaseId: db.$id,
+            databaseName: db.name,
+            backupFileId: '',
+            manifestFileId: '',
+            collectionCount: 0,
+            documentCount: 0,
+            sizeBytes: 0,
+            status: 'failed',
+            error: errorMsg
+          });
+        }
+      }
+    }
+
+    // Phase 2: Backup ALL storage buckets
+    if (!options.skipBuckets) {
+      MessageFormatter.info("Phase 2: Backing up ALL storage buckets", { prefix: "Backup" });
+
+      const allBuckets = await storage.listBuckets();
+      const bucketsToBackup = allBuckets.buckets.filter(b => b.$id !== backupBucketId);
+
+      MessageFormatter.info(`Found ${bucketsToBackup.length} buckets to backup`, { prefix: "Backup" });
+
+      for (const bucket of bucketsToBackup) {
+        try {
+          if (options.onProgress) {
+            options.onProgress(`Backing up bucket: ${bucket.name}`);
+          }
+
+          MessageFormatter.info(`Backing up bucket: ${bucket.name} (${bucket.$id})`, { prefix: "Backup" });
+
+          const bucketBackupResult = await backupBucket(
+            storage,
+            bucket.$id,
+            backupBucketId,
+            {
+              parallelDownloads: options.parallelDownloads || 10,
+              onProgress: (current, total, fileName) => {
+                if (options.onProgress) {
+                  options.onProgress(`Downloading ${fileName} (${current}/${total})`);
+                }
+              }
+            }
+          );
+
+          bucketBackups.push({
+            bucketId: bucket.$id,
+            bucketName: bucket.name,
+            backupFileId: bucketBackupResult.backupFileId,
+            manifestFileId: bucketBackupResult.manifestFileId,
+            fileCount: bucketBackupResult.fileCount,
+            sizeBytes: bucketBackupResult.totalSizeBytes,
+            status: bucketBackupResult.status,
+            error: bucketBackupResult.errors?.join('; ')
+          });
+
+          totalSizeBytes += bucketBackupResult.zipSizeBytes;
+
+          // Record individual bucket backup in tracking
+          await recordCentralizedBackup(adapter, options.trackingDatabaseId, {
+            backupType: 'bucket',
+            backupId: bucketBackupResult.backupFileId,
+            manifestFileId: bucketBackupResult.manifestFileId,
+            format: 'zip',
+            sizeBytes: bucketBackupResult.zipSizeBytes,
+            bucketId: bucket.$id,
+            fileCount: bucketBackupResult.fileCount,
+            status: bucketBackupResult.status,
+            error: bucketBackupResult.errors?.join('; '),
+            restorationStatus: 'not_restored'
+          });
+
+          MessageFormatter.success(`Bucket ${bucket.name} backed up successfully`, { prefix: "Backup" });
+        } catch (error) {
+          const errorMsg = `Failed to backup bucket ${bucket.name}: ${error instanceof Error ? error.message : String(error)}`;
+          errors.push(errorMsg);
+          logger.error(errorMsg);
+
+          bucketBackups.push({
+            bucketId: bucket.$id,
+            bucketName: bucket.name,
+            backupFileId: '',
+            manifestFileId: '',
+            fileCount: 0,
+            sizeBytes: 0,
+            status: 'failed',
+            error: errorMsg
+          });
+        }
+      }
+    }
+
+    // Phase 3: Create comprehensive manifest
+    MessageFormatter.info("Creating comprehensive backup manifest", { prefix: "Backup" });
+
+    const comprehensiveStatus: 'completed' | 'partial' | 'failed' =
+      errors.length === 0 ? 'completed' :
+      (databaseBackups.length > 0 || bucketBackups.length > 0) ? 'partial' :
+      'failed';
+
+    const manifest: ComprehensiveManifest = {
+      version: "1.0",
+      backupId,
+      createdAt: new Date().toISOString(),
+      databases: databaseBackups,
+      buckets: bucketBackups,
+      totalSizeBytes,
+      status: comprehensiveStatus,
+      errors: errors.length > 0 ? errors : undefined
+    };
+
+    // Upload comprehensive manifest
+    const manifestFileName = `comprehensive-${backupId}.json`;
+    const manifestBuffer = Buffer.from(JSON.stringify(manifest, null, 2), 'utf-8');
+    const manifestFile = await storage.createFile(
+      backupBucketId,
+      ID.unique(),
+      InputFile.fromBuffer(new Uint8Array(manifestBuffer), manifestFileName)
+    );
+
+    // Record comprehensive backup in tracking
+    await recordCentralizedBackup(adapter, options.trackingDatabaseId, {
+      backupType: 'comprehensive',
+      backupId,
+      manifestFileId: manifestFile.$id,
+      format: 'zip',
+      sizeBytes: totalSizeBytes,
+      comprehensiveBackupId: backupId,
+      status: comprehensiveStatus,
+      error: errors.length > 0 ? errors.join('; ') : undefined,
+      restorationStatus: 'not_restored'
+    });
+
+    const duration = Date.now() - startTime;
+    MessageFormatter.success(
+      `Comprehensive backup ${comprehensiveStatus} in ${(duration / 1000).toFixed(2)}s`,
+      { prefix: "Backup" }
+    );
+
+    MessageFormatter.info(
+      `Backed up ${databaseBackups.length} databases and ${bucketBackups.length} buckets (${MessageFormatter.formatBytes(totalSizeBytes)})`,
+      { prefix: "Backup" }
+    );
+
+    return {
+      backupId,
+      manifestFileId: manifestFile.$id,
+      databaseBackups,
+      bucketBackups,
+      totalSizeBytes,
+      status: comprehensiveStatus,
+      errors
+    };
+  } catch (error) {
+    const errorMsg = `Comprehensive backup failed: ${error instanceof Error ? error.message : String(error)}`;
+    MessageFormatter.error(errorMsg, error instanceof Error ? error : new Error(errorMsg), { prefix: "Backup" });
+
+    return {
+      backupId,
+      manifestFileId: '',
+      databaseBackups,
+      bucketBackups,
+      totalSizeBytes,
+      status: 'failed',
+      errors: [errorMsg, ...errors]
+    };
+  }
+}

package/src/backups/schemas/bucketManifest.ts

@@ -0,0 +1,78 @@
+import { z } from "zod";
+
+/**
+ * Schema for bucket backup manifest
+ *
+ * This manifest is stored alongside bucket backup ZIPs to describe
+ * the bucket configuration and files that were backed up.
+ */
+
+export interface BucketFileMetadata {
+  $id: string;
+  name: string;
+  size: number;
+  mimeType: string;
+  $permissions: string[];
+  chunksCount: number;
+  signature: string;
+  $createdAt: string;
+  $updatedAt: string;
+}
+
+export interface BucketConfiguration {
+  $permissions: string[];
+  fileSecurity: boolean;
+  enabled: boolean;
+  maximumFileSize: number;
+  allowedFileExtensions: string[];
+  compression: string;
+  encryption: boolean;
+  antivirus: boolean;
+}
+
+export interface BucketManifest {
+  version: string;
+  bucketId: string;
+  bucketName: string;
+  createdAt: string;
+  fileCount: number;
+  totalSizeBytes: number;
+  compression: 'gzip' | 'none';
+  files: BucketFileMetadata[];
+  bucketConfiguration: BucketConfiguration;
+}
+
+export const BucketFileMetadataSchema = z.object({
+  $id: z.string(),
+  name: z.string(),
+  size: z.number(),
+  mimeType: z.string(),
+  $permissions: z.array(z.string()),
+  chunksCount: z.number(),
+  signature: z.string(),
+  $createdAt: z.string(),
+  $updatedAt: z.string()
+});
+
+export const BucketConfigurationSchema = z.object({
+  $permissions: z.array(z.string()),
+  fileSecurity: z.boolean(),
+  enabled: z.boolean(),
+  maximumFileSize: z.number(),
+  allowedFileExtensions: z.array(z.string()),
+  compression: z.string(),
+  encryption: z.boolean(),
+  antivirus: z.boolean()
+});
+
+export const BucketManifestSchema = z.object({
+  version: z.string().default("1.0"),
+  bucketId: z.string(),
+  bucketName: z.string(),
+  createdAt: z.string(),
+  fileCount: z.number(),
+  totalSizeBytes: z.number(),
+  compression: z.enum(['gzip', 'none']),
+  files: z.array(BucketFileMetadataSchema),
+  bucketConfiguration: BucketConfigurationSchema
+});

package/src/backups/schemas/comprehensiveManifest.ts

@@ -0,0 +1,76 @@
+import { z } from "zod";
+
+/**
+ * Schema for comprehensive backup manifest
+ *
+ * This manifest tracks all databases and buckets included in a comprehensive backup,
+ * allowing for complete system restoration.
+ */
+
+export interface DatabaseBackupReference {
+  databaseId: string;
+  databaseName: string;
+  backupFileId: string;      // Storage file ID for the database backup ZIP/JSON
+  manifestFileId: string;     // Storage file ID for the database manifest JSON
+  collectionCount: number;
+  documentCount: number;
+  sizeBytes: number;
+  status: 'completed' | 'partial' | 'failed';
+  error?: string;
+}
+
+export interface BucketBackupReference {
+  bucketId: string;
+  bucketName: string;
+  backupFileId: string;       // Storage file ID for the bucket backup ZIP
+  manifestFileId: string;     // Storage file ID for the bucket manifest JSON
+  fileCount: number;
+  sizeBytes: number;
+  status: 'completed' | 'partial' | 'failed';
+  error?: string;
+}
+
+export interface ComprehensiveManifest {
+  version: string;
+  backupId: string;           // Unique ID for this comprehensive backup
+  createdAt: string;          // ISO timestamp
+  databases: DatabaseBackupReference[];
+  buckets: BucketBackupReference[];
+  totalSizeBytes: number;
+  status: 'completed' | 'partial' | 'failed';
+  errors?: string[];
+}
+
+export const DatabaseBackupReferenceSchema = z.object({
+  databaseId: z.string(),
+  databaseName: z.string(),
+  backupFileId: z.string(),
+  manifestFileId: z.string(),
+  collectionCount: z.number(),
+  documentCount: z.number(),
+  sizeBytes: z.number(),
+  status: z.enum(['completed', 'partial', 'failed']),
+  error: z.string().optional()
+});
+
+export const BucketBackupReferenceSchema = z.object({
+  bucketId: z.string(),
+  bucketName: z.string(),
+  backupFileId: z.string(),
+  manifestFileId: z.string(),
+  fileCount: z.number(),
+  sizeBytes: z.number(),
+  status: z.enum(['completed', 'partial', 'failed']),
+  error: z.string().optional()
+});
+
+export const ComprehensiveManifestSchema = z.object({
+  version: z.string().default("1.0"),
+  backupId: z.string(),
+  createdAt: z.string(),
+  databases: z.array(DatabaseBackupReferenceSchema),
+  buckets: z.array(BucketBackupReferenceSchema),
+  totalSizeBytes: z.number(),
+  status: z.enum(['completed', 'partial', 'failed']),
+  errors: z.array(z.string()).optional()
+});