@njdamstra/appwrite-utils-cli 1.8.9

package/src/adapters/index.ts

@@ -0,0 +1,37 @@
+/**
+ * Adapters Module - Unified Database API for Appwrite
+ * 
+ * This module exports all adapter classes and utilities for working with
+ * both legacy Appwrite (collections/documents) and TablesDB (tables/rows) APIs.
+ */
+
+export { AdapterFactory, type AdapterFactoryConfig, type AdapterFactoryResult } from './AdapterFactory.js';
+export { 
+  type DatabaseAdapter, 
+  type CreateRowParams,
+  type UpdateRowParams,
+  type ListRowsParams,
+  type DeleteRowParams,
+  type CreateTableParams,
+  type UpdateTableParams,
+  type ListTablesParams,
+  type DeleteTableParams,
+  type GetTableParams,
+  type BulkCreateRowsParams,
+  type BulkUpsertRowsParams,
+  type BulkDeleteRowsParams,
+  type CreateIndexParams,
+  type ListIndexesParams,
+  type DeleteIndexParams,
+  type CreateAttributeParams,
+  type UpdateAttributeParams,
+  type DeleteAttributeParams,
+  type ApiResponse,
+  type AdapterMetadata,
+  BaseAdapter 
+} from './DatabaseAdapter.js';
+export { TablesDBAdapter } from './TablesDBAdapter.js';
+export { LegacyAdapter } from './LegacyAdapter.js';
+
+// Convenience exports
+export { createDatabaseAdapter, getApiCapabilities } from './AdapterFactory.js';

package/src/backups/operations/bucketBackup.ts

@@ -0,0 +1,277 @@
+import type { Storage, Models } from "node-appwrite";
+import JSZip from "jszip";
+import { ID, Query } from "node-appwrite";
+import { InputFile } from "node-appwrite/file";
+import pLimit from "p-limit";
+import { MessageFormatter } from "../../shared/messageFormatter.js";
+import { logger } from "../../shared/logging.js";
+import type { BucketManifest, BucketFileMetadata } from "../schemas/bucketManifest.js";
+import { ulid } from "ulidx";
+
+export interface BucketBackupOptions {
+  compressionLevel?: number; // 0-9, default 6
+  parallelDownloads?: number; // Number of concurrent downloads, default 10
+  onProgress?: (current: number, total: number, fileName: string) => void;
+}
+
+export interface BucketBackupResult {
+  backupFileId: string;
+  manifestFileId: string;
+  fileCount: number;
+  totalSizeBytes: number;
+  zipSizeBytes: number;
+  status: 'completed' | 'partial' | 'failed';
+  errors?: string[];
+}
+
+/**
+ * Downloads all files from a bucket in parallel and creates a ZIP backup
+ */
+export async function backupBucket(
+  storage: Storage,
+  bucketId: string,
+  backupBucketId: string,
+  options: BucketBackupOptions = {}
+): Promise<BucketBackupResult> {
+  const {
+    compressionLevel = 6,
+    parallelDownloads = 10,
+    onProgress
+  } = options;
+
+  const errors: string[] = [];
+  let totalSizeBytes = 0;
+
+  try {
+    // Step 1: Get bucket metadata
+    MessageFormatter.info(`Fetching bucket metadata for ${bucketId}`, { prefix: "Backup" });
+    const bucket = await storage.getBucket(bucketId);
+
+    // Step 2: List all files in bucket with pagination
+    MessageFormatter.info(`Listing all files in bucket ${bucketId}`, { prefix: "Backup" });
+    const allFiles: Models.File[] = [];
+    let lastFileId: string | undefined;
+
+    while (true) {
+      const queries = [Query.limit(100)];
+      if (lastFileId) {
+        queries.push(Query.cursorAfter(lastFileId));
+      }
+
+      const filesPage = await storage.listFiles(bucketId, queries);
+      allFiles.push(...filesPage.files);
+
+      if (filesPage.files.length < 100) break;
+      lastFileId = filesPage.files[filesPage.files.length - 1].$id;
+    }
+
+    MessageFormatter.info(`Found ${allFiles.length} files to backup`, { prefix: "Backup" });
+
+    if (allFiles.length === 0) {
+      // Empty bucket - create minimal backup
+      const manifest: BucketManifest = {
+        version: "1.0",
+        bucketId: bucket.$id,
+        bucketName: bucket.name,
+        createdAt: new Date().toISOString(),
+        fileCount: 0,
+        totalSizeBytes: 0,
+        compression: bucket.compression === 'gzip' ? 'gzip' : 'none',
+        files: [],
+        bucketConfiguration: {
+          $permissions: bucket.$permissions,
+          fileSecurity: bucket.fileSecurity,
+          enabled: bucket.enabled,
+          maximumFileSize: bucket.maximumFileSize,
+          allowedFileExtensions: bucket.allowedFileExtensions,
+          compression: bucket.compression,
+          encryption: bucket.encryption,
+          antivirus: bucket.antivirus
+        }
+      };
+
+      const manifestFileId = await uploadManifest(storage, backupBucketId, bucketId, manifest);
+
+      return {
+        backupFileId: '',
+        manifestFileId,
+        fileCount: 0,
+        totalSizeBytes: 0,
+        zipSizeBytes: 0,
+        status: 'completed'
+      };
+    }
+
+    // Step 3: Download all files in parallel with concurrency limit
+    MessageFormatter.info(`Downloading ${allFiles.length} files in parallel (max ${parallelDownloads} concurrent)`, { prefix: "Backup" });
+
+    const limit = pLimit(parallelDownloads);
+    const downloadedFiles: Map<string, { buffer: Buffer; file: Models.File }> = new Map();
+    let successCount = 0;
+    let errorCount = 0;
+
+    const downloadTasks = allFiles.map((file, index) =>
+      limit(async () => {
+        try {
+          const fileBuffer = await storage.getFileDownload(bucketId, file.$id);
+          const buffer = Buffer.from(fileBuffer as ArrayBuffer);
+
+          downloadedFiles.set(file.$id, { buffer, file });
+          successCount++;
+          totalSizeBytes += file.sizeOriginal || buffer.length;
+
+          if (onProgress) {
+            onProgress(successCount + errorCount, allFiles.length, file.name);
+          }
+
+          logger.debug(`Downloaded file ${file.name}`, {
+            fileId: file.$id,
+            size: buffer.length
+          });
+        } catch (error) {
+          errorCount++;
+          const errorMsg = `Failed to download file ${file.name} (${file.$id}): ${error instanceof Error ? error.message : String(error)}`;
+          errors.push(errorMsg);
+          logger.error(errorMsg);
+
+          if (onProgress) {
+            onProgress(successCount + errorCount, allFiles.length, file.name);
+          }
+        }
+      })
+    );
+
+    await Promise.all(downloadTasks);
+
+    if (successCount === 0) {
+      throw new Error(`Failed to download any files from bucket ${bucketId}`);
+    }
+
+    MessageFormatter.info(`Successfully downloaded ${successCount}/${allFiles.length} files`, { prefix: "Backup" });
+
+    // Step 4: Create ZIP archive with all files
+    MessageFormatter.info(`Creating ZIP archive for bucket ${bucketId}`, { prefix: "Backup" });
+    const zip = new JSZip();
+
+    for (const [fileId, { buffer, file }] of downloadedFiles.entries()) {
+      // Preserve file structure by using file name
+      const fileName = file.name || `file_${fileId}`;
+      zip.file(fileName, new Uint8Array(buffer));
+    }
+
+    // Generate ZIP buffer
+    const zipBuffer = await zip.generateAsync({
+      type: "nodebuffer",
+      compression: "DEFLATE",
+      compressionOptions: { level: compressionLevel }
+    });
+
+    const zipSizeBytes = zipBuffer.length;
+    MessageFormatter.info(`ZIP archive created: ${MessageFormatter.formatBytes(zipSizeBytes)}`, { prefix: "Backup" });
+
+    // Step 5: Create manifest
+    const fileMetadata: BucketFileMetadata[] = Array.from(downloadedFiles.values()).map(({ file }) => ({
+      $id: file.$id,
+      name: file.name,
+      size: file.sizeOriginal,
+      mimeType: file.mimeType,
+      $permissions: file.$permissions,
+      chunksCount: file.chunksTotal,
+      signature: file.signature,
+      $createdAt: file.$createdAt,
+      $updatedAt: file.$updatedAt
+    }));
+
+    const manifest: BucketManifest = {
+      version: "1.0",
+      bucketId: bucket.$id,
+      bucketName: bucket.name,
+      createdAt: new Date().toISOString(),
+      fileCount: successCount,
+      totalSizeBytes,
+      compression: bucket.compression === 'gzip' ? 'gzip' : 'none',
+      files: fileMetadata,
+      bucketConfiguration: {
+        $permissions: bucket.$permissions,
+        fileSecurity: bucket.fileSecurity,
+        enabled: bucket.enabled,
+        maximumFileSize: bucket.maximumFileSize,
+        allowedFileExtensions: bucket.allowedFileExtensions,
+        compression: bucket.compression,
+        encryption: bucket.encryption,
+        antivirus: bucket.antivirus
+      }
+    };
+
+    // Step 6: Upload backup ZIP to backup bucket
+    MessageFormatter.info(`Uploading backup ZIP to bucket ${backupBucketId}`, { prefix: "Backup" });
+    const backupFileName = `${bucketId}.zip`;
+    const backupFile = await storage.createFile(
+      backupBucketId,
+      ID.unique(),
+      InputFile.fromBuffer(new Uint8Array(zipBuffer), backupFileName)
+    );
+
+    // Step 7: Upload manifest JSON
+    const manifestFileId = await uploadManifest(storage, backupBucketId, bucketId, manifest);
+
+    const status: 'completed' | 'partial' | 'failed' =
+      errorCount === 0 ? 'completed' :
+      successCount > 0 ? 'partial' :
+      'failed';
+
+    MessageFormatter.success(
+      `Bucket backup ${status}: ${successCount}/${allFiles.length} files backed up`,
+      { prefix: "Backup" }
+    );
+
+    return {
+      backupFileId: backupFile.$id,
+      manifestFileId,
+      fileCount: successCount,
+      totalSizeBytes,
+      zipSizeBytes,
+      status,
+      errors: errors.length > 0 ? errors : undefined
+    };
+  } catch (error) {
+    const errorMsg = `Bucket backup failed: ${error instanceof Error ? error.message : String(error)}`;
+    MessageFormatter.error(errorMsg, error instanceof Error ? error : new Error(errorMsg), { prefix: "Backup" });
+
+    return {
+      backupFileId: '',
+      manifestFileId: '',
+      fileCount: 0,
+      totalSizeBytes: 0,
+      zipSizeBytes: 0,
+      status: 'failed',
+      errors: [errorMsg, ...errors]
+    };
+  }
+}
+
+/**
+ * Uploads bucket manifest JSON to backup bucket
+ */
+async function uploadManifest(
+  storage: Storage,
+  backupBucketId: string,
+  bucketId: string,
+  manifest: BucketManifest
+): Promise<string> {
+  const manifestFileName = `${bucketId}.json`;
+  const manifestBuffer = Buffer.from(JSON.stringify(manifest, null, 2), 'utf-8');
+
+  const manifestFile = await storage.createFile(
+    backupBucketId,
+    ID.unique(),
+    InputFile.fromBuffer(new Uint8Array(manifestBuffer), manifestFileName)
+  );
+
+  logger.info("Uploaded bucket manifest", {
+    manifestFileId: manifestFile.$id,
+    bucketId
+  });
+
+  return manifestFile.$id;
+}

package/src/backups/operations/collectionBackup.ts

@@ -0,0 +1,310 @@
+import type { Storage, Databases, Models } from "node-appwrite";
+import { ID, Query } from "node-appwrite";
+import { InputFile } from "node-appwrite/file";
+import { ulid } from "ulidx";
+import { MessageFormatter } from "../../shared/messageFormatter.js";
+import { logger } from "../../shared/logging.js";
+import type { DatabaseAdapter } from "../../adapters/DatabaseAdapter.js";
+import { tryAwaitWithRetry } from "@njdamstra/appwrite-utils";
+import { splitIntoBatches } from "../../shared/migrationHelpers.js";
+import { retryFailedPromises } from "../../utils/retryFailedPromises.js";
+import { ProgressManager } from "../../shared/progressManager.js";
+import { createBackupZip } from "../../storage/backupCompression.js";
+import {
+  recordCentralizedBackup,
+  createCentralizedBackupTrackingTable
+} from "../tracking/centralizedTracking.js";
+import type { AppwriteConfig } from "@njdamstra/appwrite-utils";
+
+export interface CollectionBackupOptions {
+  trackingDatabaseId: string;
+  databaseId: string;
+  collectionIds: string[];
+  backupFormat?: 'json' | 'zip';
+  onProgress?: (message: string) => void;
+}
+
+export interface CollectionBackupResult {
+  backupId: string;
+  manifestFileId: string;
+  databaseId: string;
+  collections: Array<{
+    collectionId: string;
+    collectionName: string;
+    documentCount: number;
+    status: 'completed' | 'failed';
+    error?: string;
+  }>;
+  totalDocuments: number;
+  sizeBytes: number;
+  status: 'completed' | 'partial' | 'failed';
+  errors: string[];
+}
+
+interface BackupData {
+  database: string;
+  collections: string[];
+  documents: Array<{
+    collectionId: string;
+    data: string;
+  }>;
+}
+
+/**
+ * Backup specific collections from a database
+ */
+export async function backupCollections(
+  config: AppwriteConfig,
+  databases: Databases,
+  storage: Storage,
+  adapter: DatabaseAdapter,
+  options: CollectionBackupOptions
+): Promise<CollectionBackupResult> {
+  const startTime = Date.now();
+  const backupId = ulid();
+  const errors: string[] = [];
+  const collections: CollectionBackupResult['collections'] = [];
+  let totalDocuments = 0;
+  let totalSizeBytes = 0;
+
+  try {
+    // Ensure tracking table exists
+    await createCentralizedBackupTrackingTable(adapter, options.trackingDatabaseId);
+
+    const backupBucketId = "appwrite-backups";
+    MessageFormatter.info(`Starting collection backup ${backupId}`, { prefix: "Backup" });
+    MessageFormatter.info(`Database: ${options.databaseId}, Collections: ${options.collectionIds.length}`, { prefix: "Backup" });
+
+    // Get database info
+    const db = await tryAwaitWithRetry(
+      async () => await databases.get(options.databaseId)
+    );
+
+    const backupData: BackupData = {
+      database: JSON.stringify(db),
+      collections: [],
+      documents: []
+    };
+
+    // Phase 1: Count documents for progress tracking
+    MessageFormatter.step(1, 3, "Analyzing collections");
+    let totalItems = options.collectionIds.length; // Start with collection count
+
+    for (const collectionId of options.collectionIds) {
+      try {
+        const documentCount = await tryAwaitWithRetry(
+          async () => (await databases.listDocuments(options.databaseId, collectionId, [Query.limit(1)])).total
+        );
+        totalDocuments += documentCount;
+        totalItems += documentCount;
+      } catch (error) {
+        MessageFormatter.warning(`Could not count documents in collection ${collectionId}`);
+      }
+    }
+
+    const progress = ProgressManager.create(`backup-collections-${backupId}`, totalItems, {
+      title: `Backing up ${options.collectionIds.length} collections`,
+    });
+
+    // Phase 2: Backup selected collections
+    MessageFormatter.step(2, 3, `Processing ${options.collectionIds.length} collections and ${totalDocuments} documents`);
+
+    let processedDocuments = 0;
+
+    for (const collectionId of options.collectionIds) {
+      try {
+        if (options.onProgress) {
+          options.onProgress(`Backing up collection: ${collectionId}`);
+        }
+
+        // Get collection metadata
+        const collection = await tryAwaitWithRetry(
+          async () => await databases.getCollection(options.databaseId, collectionId)
+        );
+
+        backupData.collections.push(JSON.stringify(collection));
+        progress.increment(1, `Processing collection: ${collection.name}`);
+
+        // Backup all documents in this collection
+        let lastDocumentId = "";
+        let moreDocuments = true;
+        let collectionDocumentCount = 0;
+
+        while (moreDocuments) {
+          const documentResponse = await tryAwaitWithRetry(
+            async () =>
+              await databases.listDocuments(options.databaseId, collectionId, [
+                Query.limit(500),
+                ...(lastDocumentId ? [Query.cursorAfter(lastDocumentId)] : []),
+              ])
+          );
+
+          collectionDocumentCount += documentResponse.documents.length;
+
+          const documentPromises = documentResponse.documents.map(
+            ({ $id: documentId }) =>
+              databases.getDocument(options.databaseId, collectionId, documentId)
+          );
+
+          const promiseBatches = splitIntoBatches(documentPromises);
+          const documentsPulled = [];
+
+          for (const batch of promiseBatches) {
+            const successfulDocuments = await retryFailedPromises(batch);
+            documentsPulled.push(...successfulDocuments);
+
+            // Update progress for each batch
+            progress.increment(successfulDocuments.length,
+              `Processing ${collection.name}: ${processedDocuments + successfulDocuments.length}/${totalDocuments} documents`
+            );
+            processedDocuments += successfulDocuments.length;
+          }
+
+          backupData.documents.push({
+            collectionId: collectionId,
+            data: JSON.stringify(documentsPulled),
+          });
+
+          moreDocuments = documentResponse.documents.length === 500;
+          if (moreDocuments) {
+            lastDocumentId = documentResponse.documents[documentResponse.documents.length - 1].$id;
+          }
+        }
+
+        collections.push({
+          collectionId,
+          collectionName: collection.name,
+          documentCount: collectionDocumentCount,
+          status: 'completed'
+        });
+
+        MessageFormatter.success(
+          `Collection ${collection.name} backed up with ${MessageFormatter.formatNumber(collectionDocumentCount)} documents`
+        );
+      } catch (error) {
+        const errorMsg = `Failed to backup collection ${collectionId}: ${error instanceof Error ? error.message : String(error)}`;
+        errors.push(errorMsg);
+        logger.error(errorMsg);
+
+        collections.push({
+          collectionId,
+          collectionName: collectionId,
+          documentCount: 0,
+          status: 'failed',
+          error: errorMsg
+        });
+      }
+    }
+
+    // Phase 3: Create backup file
+    MessageFormatter.step(3, 3, "Creating backup file");
+
+    let inputFile: any;
+    let fileName: string;
+    let backupSize: number;
+
+    if (options.backupFormat === 'zip') {
+      // Create compressed backup
+      const zipBuffer = await createBackupZip(backupData);
+      fileName = `${new Date().toISOString()}-${options.databaseId}-collections.zip`;
+      backupSize = zipBuffer.length;
+      inputFile = InputFile.fromBuffer(new Uint8Array(zipBuffer), fileName);
+    } else {
+      // Use JSON format
+      const backupDataString = JSON.stringify(backupData, null, 2);
+      fileName = `${new Date().toISOString()}-${options.databaseId}-collections.json`;
+      backupSize = Buffer.byteLength(backupDataString, 'utf8');
+      inputFile = InputFile.fromPlainText(backupDataString, fileName);
+    }
+
+    const fileCreated = await storage.createFile(
+      backupBucketId,
+      ulid(),
+      inputFile
+    );
+
+    totalSizeBytes = backupSize;
+
+    // Create manifest
+    const manifestData = {
+      version: "1.0",
+      backupId,
+      databaseId: options.databaseId,
+      collectionIds: options.collectionIds,
+      collections: collections,
+      format: options.backupFormat || 'json',
+      createdAt: new Date().toISOString(),
+      totalDocuments: processedDocuments,
+      totalSizeBytes: backupSize
+    };
+
+    const manifestBuffer = Buffer.from(JSON.stringify(manifestData, null, 2), 'utf-8');
+    const manifestFile = await storage.createFile(
+      backupBucketId,
+      ID.unique(),
+      InputFile.fromBuffer(new Uint8Array(manifestBuffer), `${backupId}-manifest.json`)
+    );
+
+    // Record in centralized tracking
+    await recordCentralizedBackup(adapter, options.trackingDatabaseId, {
+      backupType: 'database',
+      backupId: fileCreated.$id,
+      manifestFileId: manifestFile.$id,
+      format: options.backupFormat || 'json',
+      sizeBytes: backupSize,
+      databaseId: options.databaseId,
+      collections: backupData.collections.length,
+      documents: processedDocuments,
+      status: errors.length === 0 ? 'completed' : 'partial',
+      error: errors.length > 0 ? errors.join('; ') : undefined,
+      restorationStatus: 'not_restored'
+    });
+
+    progress.stop();
+
+    const duration = Date.now() - startTime;
+    const status: 'completed' | 'partial' | 'failed' =
+      errors.length === 0 ? 'completed' :
+      collections.some(c => c.status === 'completed') ? 'partial' :
+      'failed';
+
+    MessageFormatter.success(
+      `Collection backup ${status} in ${(duration / 1000).toFixed(2)}s`,
+      { prefix: "Backup" }
+    );
+
+    MessageFormatter.operationSummary("Collection Backup", {
+      database: options.databaseId,
+      collections: backupData.collections.length,
+      documents: processedDocuments,
+      fileSize: MessageFormatter.formatBytes(backupSize),
+      backupFile: fileName,
+      bucket: backupBucketId,
+    }, duration);
+
+    return {
+      backupId,
+      manifestFileId: manifestFile.$id,
+      databaseId: options.databaseId,
+      collections,
+      totalDocuments: processedDocuments,
+      sizeBytes: totalSizeBytes,
+      status,
+      errors
+    };
+  } catch (error) {
+    const errorMsg = `Collection backup failed: ${error instanceof Error ? error.message : String(error)}`;
+    MessageFormatter.error(errorMsg, error instanceof Error ? error : new Error(errorMsg), { prefix: "Backup" });
+
+    return {
+      backupId,
+      manifestFileId: '',
+      databaseId: options.databaseId,
+      collections: [],
+      totalDocuments: 0,
+      sizeBytes: 0,
+      status: 'failed',
+      errors: [errorMsg, ...errors]
+    };
+  }
+}