appwrite-utils-cli 1.5.2 → 1.6.1

package/src/backups/schemas/bucketManifest.ts

@@ -0,0 +1,78 @@
+import { z } from "zod";
+
+/**
+ * Schema for bucket backup manifest
+ *
+ * This manifest is stored alongside bucket backup ZIPs to describe
+ * the bucket configuration and files that were backed up.
+ */
+
+export interface BucketFileMetadata {
+  $id: string;
+  name: string;
+  size: number;
+  mimeType: string;
+  $permissions: string[];
+  chunksCount: number;
+  signature: string;
+  $createdAt: string;
+  $updatedAt: string;
+}
+
+export interface BucketConfiguration {
+  $permissions: string[];
+  fileSecurity: boolean;
+  enabled: boolean;
+  maximumFileSize: number;
+  allowedFileExtensions: string[];
+  compression: string;
+  encryption: boolean;
+  antivirus: boolean;
+}
+
+export interface BucketManifest {
+  version: string;
+  bucketId: string;
+  bucketName: string;
+  createdAt: string;
+  fileCount: number;
+  totalSizeBytes: number;
+  compression: 'gzip' | 'none';
+  files: BucketFileMetadata[];
+  bucketConfiguration: BucketConfiguration;
+}
+
+export const BucketFileMetadataSchema = z.object({
+  $id: z.string(),
+  name: z.string(),
+  size: z.number(),
+  mimeType: z.string(),
+  $permissions: z.array(z.string()),
+  chunksCount: z.number(),
+  signature: z.string(),
+  $createdAt: z.string(),
+  $updatedAt: z.string()
+});
+
+export const BucketConfigurationSchema = z.object({
+  $permissions: z.array(z.string()),
+  fileSecurity: z.boolean(),
+  enabled: z.boolean(),
+  maximumFileSize: z.number(),
+  allowedFileExtensions: z.array(z.string()),
+  compression: z.string(),
+  encryption: z.boolean(),
+  antivirus: z.boolean()
+});
+
+export const BucketManifestSchema = z.object({
+  version: z.string().default("1.0"),
+  bucketId: z.string(),
+  bucketName: z.string(),
+  createdAt: z.string(),
+  fileCount: z.number(),
+  totalSizeBytes: z.number(),
+  compression: z.enum(['gzip', 'none']),
+  files: z.array(BucketFileMetadataSchema),
+  bucketConfiguration: BucketConfigurationSchema
+});

package/src/backups/schemas/comprehensiveManifest.ts

@@ -0,0 +1,76 @@
+import { z } from "zod";
+
+/**
+ * Schema for comprehensive backup manifest
+ *
+ * This manifest tracks all databases and buckets included in a comprehensive backup,
+ * allowing for complete system restoration.
+ */
+
+export interface DatabaseBackupReference {
+  databaseId: string;
+  databaseName: string;
+  backupFileId: string;      // Storage file ID for the database backup ZIP/JSON
+  manifestFileId: string;     // Storage file ID for the database manifest JSON
+  collectionCount: number;
+  documentCount: number;
+  sizeBytes: number;
+  status: 'completed' | 'partial' | 'failed';
+  error?: string;
+}
+
+export interface BucketBackupReference {
+  bucketId: string;
+  bucketName: string;
+  backupFileId: string;       // Storage file ID for the bucket backup ZIP
+  manifestFileId: string;     // Storage file ID for the bucket manifest JSON
+  fileCount: number;
+  sizeBytes: number;
+  status: 'completed' | 'partial' | 'failed';
+  error?: string;
+}
+
+export interface ComprehensiveManifest {
+  version: string;
+  backupId: string;           // Unique ID for this comprehensive backup
+  createdAt: string;          // ISO timestamp
+  databases: DatabaseBackupReference[];
+  buckets: BucketBackupReference[];
+  totalSizeBytes: number;
+  status: 'completed' | 'partial' | 'failed';
+  errors?: string[];
+}
+
+export const DatabaseBackupReferenceSchema = z.object({
+  databaseId: z.string(),
+  databaseName: z.string(),
+  backupFileId: z.string(),
+  manifestFileId: z.string(),
+  collectionCount: z.number(),
+  documentCount: z.number(),
+  sizeBytes: z.number(),
+  status: z.enum(['completed', 'partial', 'failed']),
+  error: z.string().optional()
+});
+
+export const BucketBackupReferenceSchema = z.object({
+  bucketId: z.string(),
+  bucketName: z.string(),
+  backupFileId: z.string(),
+  manifestFileId: z.string(),
+  fileCount: z.number(),
+  sizeBytes: z.number(),
+  status: z.enum(['completed', 'partial', 'failed']),
+  error: z.string().optional()
+});
+
+export const ComprehensiveManifestSchema = z.object({
+  version: z.string().default("1.0"),
+  backupId: z.string(),
+  createdAt: z.string(),
+  databases: z.array(DatabaseBackupReferenceSchema),
+  buckets: z.array(BucketBackupReferenceSchema),
+  totalSizeBytes: z.number(),
+  status: z.enum(['completed', 'partial', 'failed']),
+  errors: z.array(z.string()).optional()
+});

package/src/backups/tracking/centralizedTracking.ts

@@ -0,0 +1,352 @@
+import type { DatabaseAdapter } from "../../adapters/DatabaseAdapter.js";
+import { logger } from "../../shared/logging.js";
+import { tryAwaitWithRetry } from "../../utils/helperFunctions.js";
+import { Query, ID } from "node-appwrite";
+import {
+  BACKUP_TABLE_ID,
+  BACKUP_TABLE_NAME,
+  type BackupMetadata,
+  type BackupType,
+  BackupMetadataSchema
+} from "../../shared/backupMetadataSchema.js";
+
+/**
+ * Centralized backup tracking system
+ *
+ * All backups (databases, buckets, comprehensive) are tracked in a single
+ * database selected by the user, providing a centralized backup registry.
+ */
+
+/**
+ * Checks if backup tracking table exists in database
+ */
+async function tableExists(
+  db: DatabaseAdapter,
+  databaseId: string
+): Promise<boolean> {
+  try {
+    await db.getTable({ databaseId, tableId: BACKUP_TABLE_ID });
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+
+/**
+ * Creates the centralized backup tracking table with enhanced schema
+ */
+export async function createCentralizedBackupTrackingTable(
+  db: DatabaseAdapter,
+  databaseId: string
+): Promise<void> {
+  // Check if table already exists
+  const exists = await tableExists(db, databaseId);
+
+  if (exists) {
+    // Table exists - validate its schema
+    try {
+      const tableData = await db.getTable({ databaseId, tableId: BACKUP_TABLE_ID });
+      const existingAttrs = ((tableData.data as any).attributes || []);
+
+      // Expected attribute keys from our schema
+      const expectedAttrKeys = [
+        "backupType", "backupId", "manifestFileId", "format", "sizeBytes",
+        "databaseId", "bucketId", "comprehensiveBackupId",
+        "collections", "documents", "fileCount",
+        "status", "error",
+        "restoredAt", "restorationStatus", "restorationError"
+      ];
+
+      // Existing attribute keys
+      const existingAttrKeys = new Set(existingAttrs.map((a: any) => a.key));
+
+      // Check if they match
+      const hasAllAttributes = expectedAttrKeys.every(key => existingAttrKeys.has(key));
+
+      // Check if any have wrong sizes (old 10000 vs new 1000)
+      const hasWrongSizes = existingAttrs.some((a: any) =>
+        (a.key === 'error' || a.key === 'restorationError') && a.size === 10000
+      );
+
+      if (!hasAllAttributes || hasWrongSizes) {
+        logger.warn("Backup table exists but has incorrect schema - recreating", {
+          tableId: BACKUP_TABLE_ID,
+          hasAllAttributes,
+          hasWrongSizes,
+          existingCount: existingAttrKeys.size,
+          expectedCount: expectedAttrKeys.length
+        });
+
+        // Delete the old table
+        await db.deleteTable({ databaseId, tableId: BACKUP_TABLE_ID });
+
+        logger.info("Old backup table deleted, will recreate with correct schema");
+      } else {
+        logger.debug("Backup table exists with correct schema", {
+          tableId: BACKUP_TABLE_ID,
+          attributeCount: existingAttrKeys.size
+        });
+        return; // Table is good, no need to recreate
+      }
+    } catch (error) {
+      // Error checking table - continue to create
+      logger.debug("Error checking existing table, will attempt to create", {
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
+
+  // Create table (either first time or after deletion)
+  logger.info("Creating centralized backup tracking table", {
+    databaseId,
+    tableId: BACKUP_TABLE_ID
+  });
+
+  await tryAwaitWithRetry(async () => {
+    await db.createTable({
+      databaseId,
+      id: BACKUP_TABLE_ID,
+      name: BACKUP_TABLE_NAME,
+      permissions: [],
+      documentSecurity: false,
+      enabled: true
+    });
+  });
+
+  // Define all attributes for the enhanced schema
+  const attributes = [
+    // Core backup info
+    { key: "backupType", type: "enum" as const, elements: ["database", "bucket", "comprehensive"], required: true },
+    { key: "backupId", type: "string" as const, size: 50, required: true },
+    { key: "manifestFileId", type: "string" as const, size: 50, required: false },
+    { key: "format", type: "enum" as const, elements: ["json", "zip"], required: true },
+    { key: "sizeBytes", type: "integer" as const, required: true },
+
+    // Resource identification (optional, at least one present based on type)
+    { key: "databaseId", type: "string" as const, size: 50, required: false },
+    { key: "bucketId", type: "string" as const, size: 50, required: false },
+    { key: "comprehensiveBackupId", type: "string" as const, size: 50, required: false },
+
+    // Database-specific metrics
+    { key: "collections", type: "integer" as const, required: false },
+    { key: "documents", type: "integer" as const, required: false },
+
+    // Bucket-specific metrics
+    { key: "fileCount", type: "integer" as const, required: false },
+
+    // Status tracking
+    { key: "status", type: "enum" as const, elements: ["completed", "partial", "failed"], required: true },
+    { key: "error", type: "string" as const, size: 1000, required: false },
+
+    // Restoration tracking
+    { key: "restoredAt", type: "string" as const, size: 50, required: false },
+    { key: "restorationStatus", type: "enum" as const, elements: ["completed", "partial", "failed", "not_restored"], required: false },
+    { key: "restorationError", type: "string" as const, size: 1000, required: false }
+  ];
+
+  // Create each attribute with retry logic
+  // No need to check for existing attributes - table is freshly created or validated above
+  for (const attr of attributes) {
+    await tryAwaitWithRetry(async () => {
+      await db.createAttribute({
+        databaseId,
+        tableId: BACKUP_TABLE_ID,
+        ...attr
+      });
+    });
+  }
+
+  logger.info("Centralized backup tracking table created successfully", {
+    databaseId,
+    tableId: BACKUP_TABLE_ID,
+    attributeCount: attributes.length
+  });
+}
+
+/**
+ * Records backup metadata in the centralized tracking table
+ */
+export async function recordCentralizedBackup(
+  db: DatabaseAdapter,
+  trackingDatabaseId: string,
+  metadata: Omit<BackupMetadata, '$id' | '$createdAt' | '$updatedAt'>
+): Promise<BackupMetadata> {
+  // Ensure tracking table exists with correct schema
+  await createCentralizedBackupTrackingTable(db, trackingDatabaseId);
+
+  // Create backup record with all fields
+  // Table is guaranteed to have all attributes after createCentralizedBackupTrackingTable
+  const result = await db.createRow({
+    databaseId: trackingDatabaseId,
+    tableId: BACKUP_TABLE_ID,
+    id: ID.unique(),
+    data: {
+      // Core fields
+      backupType: metadata.backupType,
+      backupId: metadata.backupId,
+      manifestFileId: metadata.manifestFileId || null,
+      format: metadata.format,
+      sizeBytes: metadata.sizeBytes,
+
+      // Resource identification
+      databaseId: metadata.databaseId || null,
+      bucketId: metadata.bucketId || null,
+      comprehensiveBackupId: metadata.comprehensiveBackupId || null,
+
+      // Metrics
+      collections: metadata.collections || null,
+      documents: metadata.documents || null,
+      fileCount: metadata.fileCount || null,
+
+      // Status
+      status: metadata.status,
+      error: metadata.error || null,
+
+      // Restoration
+      restoredAt: metadata.restoredAt || null,
+      restorationStatus: metadata.restorationStatus || 'not_restored',
+      restorationError: metadata.restorationError || null
+    }
+  });
+
+  logger.info("Recorded centralized backup metadata", {
+    backupType: metadata.backupType,
+    backupId: metadata.backupId,
+    trackingDatabaseId
+  });
+
+  return result.data as BackupMetadata;
+}
+
+/**
+ * Lists all backups of a specific type, sorted by creation date (newest first)
+ */
+export async function listCentralizedBackups(
+  db: DatabaseAdapter,
+  trackingDatabaseId: string,
+  options?: {
+    backupType?: BackupType;
+    resourceId?: string;  // databaseId or bucketId
+    limit?: number;
+  }
+): Promise<BackupMetadata[]> {
+  try {
+    const queries: string[] = [
+      Query.orderDesc("$createdAt"),
+      Query.limit(options?.limit || 100)
+    ];
+
+    // Filter by backup type if specified
+    if (options?.backupType) {
+      queries.push(Query.equal("backupType", options.backupType));
+    }
+
+    // Filter by resource ID if specified
+    if (options?.resourceId) {
+      if (options.backupType === 'database') {
+        queries.push(Query.equal("databaseId", options.resourceId));
+      } else if (options.backupType === 'bucket') {
+        queries.push(Query.equal("bucketId", options.resourceId));
+      }
+    }
+
+    const result = await db.listRows({
+      databaseId: trackingDatabaseId,
+      tableId: BACKUP_TABLE_ID,
+      queries
+    });
+
+    return (result.rows || []) as BackupMetadata[];
+  } catch (error) {
+    logger.debug("No centralized backup tracking table found", { trackingDatabaseId });
+    return [];
+  }
+}
+
+/**
+ * Gets a specific backup by its backup file ID
+ */
+export async function getCentralizedBackup(
+  db: DatabaseAdapter,
+  trackingDatabaseId: string,
+  backupId: string
+): Promise<BackupMetadata | null> {
+  try {
+    const result = await db.listRows({
+      databaseId: trackingDatabaseId,
+      tableId: BACKUP_TABLE_ID,
+      queries: [
+        Query.equal("backupId", backupId),
+        Query.limit(1)
+      ]
+    });
+
+    if (result.rows && result.rows.length > 0) {
+      return result.rows[0] as BackupMetadata;
+    }
+
+    return null;
+  } catch (error) {
+    logger.debug("Backup not found", { backupId, trackingDatabaseId });
+    return null;
+  }
+}
+
+/**
+ * Updates restoration status for a backup
+ */
+export async function updateRestorationStatus(
+  db: DatabaseAdapter,
+  trackingDatabaseId: string,
+  backupRecordId: string,
+  restorationData: {
+    restoredAt: string;
+    restorationStatus: 'completed' | 'partial' | 'failed';
+    restorationError?: string;
+  }
+): Promise<void> {
+  await db.updateRow({
+    databaseId: trackingDatabaseId,
+    tableId: BACKUP_TABLE_ID,
+    id: backupRecordId,
+    data: {
+      restoredAt: restorationData.restoredAt,
+      restorationStatus: restorationData.restorationStatus,
+      restorationError: restorationData.restorationError || null
+    }
+  });
+
+  logger.info("Updated restoration status", {
+    backupRecordId,
+    restorationStatus: restorationData.restorationStatus
+  });
+}
+
+/**
+ * Gets the most recent comprehensive backup
+ */
+export async function getLastComprehensiveBackup(
+  db: DatabaseAdapter,
+  trackingDatabaseId: string
+): Promise<BackupMetadata | null> {
+  try {
+    const result = await db.listRows({
+      databaseId: trackingDatabaseId,
+      tableId: BACKUP_TABLE_ID,
+      queries: [
+        Query.equal("backupType", "comprehensive"),
+        Query.orderDesc("$createdAt"),
+        Query.limit(1)
+      ]
+    });
+
+    if (result.rows && result.rows.length > 0) {
+      return result.rows[0] as BackupMetadata;
+    }
+
+    return null;
+  } catch (error) {
+    logger.debug("No comprehensive backup found", { trackingDatabaseId });
+    return null;
+  }
+}

package/src/cli/commands/configCommands.ts

@@ -0,0 +1,194 @@
+import inquirer from "inquirer";
+import { MessageFormatter } from "../../shared/messageFormatter.js";
+import { migrateConfig } from "../../utils/configMigration.js";
+import {
+  validateCollectionsTablesConfig,
+  reportValidationResults,
+} from "../../config/configValidation.js";
+import {
+  createMigrationPlan,
+  executeMigrationPlan,
+  saveMigrationResult,
+  type MigrationStrategy,
+} from "../../config/configMigration.js";
+import { createEmptyCollection } from "../../utils/setupFiles.js";
+import chalk from "chalk";
+import type { InteractiveCLI } from "../../interactiveCLI.js";
+
+export const configCommands = {
+  async migrateTypeScriptConfig(cli: InteractiveCLI): Promise<void> {
+    try {
+      MessageFormatter.info("Starting TypeScript to YAML configuration migration...", { prefix: "Migration" });
+
+      // Perform the migration
+      await migrateConfig((cli as any).currentDir);
+
+      // Reset the detection flag
+      (cli as any).isUsingTypeScriptConfig = false;
+
+      // Reset the controller to pick up the new config
+      (cli as any).controller = undefined;
+
+      MessageFormatter.success("Migration completed successfully!", { prefix: "Migration" });
+      MessageFormatter.info("Your configuration has been migrated to the .appwrite directory structure", { prefix: "Migration" });
+      MessageFormatter.info("You can now use YAML configuration for easier management", { prefix: "Migration" });
+
+    } catch (error) {
+      MessageFormatter.error("Migration failed", error instanceof Error ? error : new Error(String(error)), { prefix: "Migration" });
+    }
+  },
+
+  async validateConfiguration(cli: InteractiveCLI): Promise<void> {
+    try {
+      MessageFormatter.info("Starting configuration validation...", { prefix: "Validation" });
+
+      await (cli as any).initControllerIfNeeded();
+      const config = (cli as any).controller?.config;
+
+      if (!config) {
+        MessageFormatter.error("No configuration found to validate", undefined, { prefix: "Validation" });
+        return;
+      }
+
+      const { validateCollectionsTablesConfig, reportValidationResults } = await import("../../config/configValidation.js");
+      const validation = validateCollectionsTablesConfig(config);
+
+      reportValidationResults(validation, { verbose: true });
+
+      if (validation.isValid) {
+        MessageFormatter.success("Configuration validation passed!", { prefix: "Validation" });
+      } else {
+        MessageFormatter.error("Configuration validation failed", undefined, { prefix: "Validation" });
+      }
+
+    } catch (error) {
+      MessageFormatter.error("Validation failed", error instanceof Error ? error : new Error(String(error)), { prefix: "Validation" });
+    }
+  },
+
+  async migrateCollectionsToTables(cli: InteractiveCLI): Promise<void> {
+    try {
+      MessageFormatter.info("Starting collections to tables migration...", { prefix: "Migration" });
+
+      await (cli as any).initControllerIfNeeded();
+
+      // Ensure config is properly loaded with YAML collections
+      if (!(cli as any).controller?.config) {
+        MessageFormatter.error("No configuration found", undefined, { prefix: "Migration" });
+        return;
+      }
+
+      // Check if collections exist
+      if (!(cli as any).controller.config.collections || (cli as any).controller.config.collections.length === 0) {
+        MessageFormatter.error("No collections found in configuration. Please check your YAML files or appwriteConfig.ts", undefined, { prefix: "Migration" });
+        return;
+      }
+
+      const { createMigrationPlan, executeMigrationPlan, saveMigrationResult } = await import("../../config/configMigration.js");
+
+      // Get user's migration strategy preference
+      const { strategy } = await inquirer.prompt([
+        {
+          type: "list",
+          name: "strategy",
+          message: "Choose migration strategy:",
+          choices: [
+            { name: "Move files (rename collections/ to tables/)", value: "move" },
+            { name: "Copy files (keep both collections/ and tables/)", value: "copy" },
+            { name: "Dual format (create both .ts and .yaml versions)", value: "dual" }
+          ]
+        }
+      ]);
+
+      // Map user-friendly strategy names to internal MigrationStrategy types
+      const migrationStrategy = strategy === "move" ? "full_migration" :
+                               strategy === "copy" || strategy === "dual" ? "dual_format" : "full_migration";
+      const plan = createMigrationPlan((cli as any).controller.config, migrationStrategy);
+
+      const { confirmed } = await inquirer.prompt([
+        {
+          type: "confirm",
+          name: "confirmed",
+          message: `Proceed with migration? This will affect ${plan.collectionsToMigrate?.length || 0} files.`,
+          default: false
+        }
+      ]);
+
+      if (confirmed) {
+        const result = executeMigrationPlan((cli as any).controller.config, plan);
+        await saveMigrationResult(result, (cli as any).currentDir);
+
+        MessageFormatter.success("Collections to tables migration completed!", { prefix: "Migration" });
+      } else {
+        MessageFormatter.info("Migration cancelled by user", { prefix: "Migration" });
+      }
+
+    } catch (error) {
+      MessageFormatter.error("Migration failed", error instanceof Error ? error : new Error(String(error)), { prefix: "Migration" });
+    }
+  },
+
+  async createCollectionConfig(cli: InteractiveCLI): Promise<void> {
+    const { collectionName } = await inquirer.prompt([
+      {
+        type: "input",
+        name: "collectionName",
+        message: chalk.blue("Enter the name of the collection:"),
+        validate: (input) =>
+          input.trim() !== "" || "Collection name cannot be empty.",
+      },
+    ]);
+    MessageFormatter.progress(`Creating collection config file for '${collectionName}'...`, { prefix: "Collections" });
+    createEmptyCollection(collectionName);
+    MessageFormatter.success(`Collection config file created for '${collectionName}'`, { prefix: "Collections" });
+  },
+
+  async reloadConfigWithSessionPreservation(cli: InteractiveCLI): Promise<void> {
+    MessageFormatter.progress("Reloading configuration files with session preservation...", { prefix: "Config" });
+    try {
+      const controller = (cli as any).controller;
+      const UtilsController = (await import("../../utilsController.js")).UtilsController;
+
+      if (controller) {
+        const sessionInfo = controller.getSessionInfo();
+
+        if (sessionInfo.hasSession) {
+          // Extract session details for preservation
+          const sessionData = {
+            sessionCookie: controller.sessionCookie,
+            sessionMetadata: controller.sessionMetadata
+          };
+
+          // Store current config values for potential directConfig creation
+          const currentConfig = controller.config;
+          let directConfig: any = undefined;
+
+          if (currentConfig?.appwriteEndpoint && currentConfig?.appwriteProject) {
+            directConfig = {
+              appwriteEndpoint: currentConfig.appwriteEndpoint,
+              appwriteProject: currentConfig.appwriteProject,
+              appwriteKey: currentConfig.appwriteKey,
+              ...sessionData // Preserve session
+            };
+          }
+
+          // Reinitialize controller with session preservation
+          (cli as any).controller = new UtilsController((cli as any).currentDir, directConfig);
+          await (cli as any).controller.init();
+
+          MessageFormatter.success("Configuration reloaded with session preserved", { prefix: "Config" });
+        } else {
+          // No session to preserve, standard reload
+          await controller.reloadConfig();
+          MessageFormatter.success("Configuration files reloaded successfully", { prefix: "Config" });
+        }
+      } else {
+        // Initialize if no controller exists
+        await (cli as any).initControllerIfNeeded();
+        MessageFormatter.success("Configuration initialized successfully", { prefix: "Config" });
+      }
+    } catch (error) {
+      MessageFormatter.error("Failed to reload configuration files", error instanceof Error ? error : new Error(String(error)), { prefix: "Config" });
+    }
+  }
+};