@njdamstra/appwrite-utils-cli 1.8.9

package/src/shared/logging.ts

@@ -0,0 +1,149 @@
+import winston from "winston";
+import fs from "fs";
+import path from "path";
+
+export interface LoggingConfig {
+  enabled: boolean;
+  level: string;
+  logDirectory?: string;
+  console: boolean;
+}
+
+/**
+ * Predefined logging configurations for common debugging scenarios
+ */
+export const LOGGING_PRESETS = {
+  /** Minimal logging - errors only to console */
+  minimal: {
+    enabled: false,
+    level: 'error',
+    console: true
+  },
+  /** Standard logging - info level to file and console */
+  standard: {
+    enabled: true,
+    level: 'info',
+    console: true
+  },
+  /** Debug logging - verbose debug output for troubleshooting */
+  debug: {
+    enabled: true,
+    level: 'debug',
+    console: true
+  },
+  /** Silent - no logging output */
+  silent: {
+    enabled: false,
+    level: 'error',
+    console: false
+  }
+} as const;
+
+const DEFAULT_LOGGING_CONFIG: LoggingConfig = {
+  enabled: false,
+  level: "info",
+  console: false,
+};
+
+let loggingConfig: LoggingConfig = DEFAULT_LOGGING_CONFIG;
+
+export const configureLogging = (config: Partial<LoggingConfig> = {}) => {
+  loggingConfig = { ...DEFAULT_LOGGING_CONFIG, ...config };
+};
+
+/**
+ * Configure logging using a preset
+ */
+export const configureLoggingPreset = (preset: keyof typeof LOGGING_PRESETS, logDirectory?: string) => {
+  const presetConfig = LOGGING_PRESETS[preset];
+  configureLogging({
+    ...presetConfig,
+    ...(logDirectory && { logDirectory })
+  });
+  updateLogger();
+};
+
+const createLogger = () => {
+  const transports: winston.transport[] = [];
+
+  // Add console transport if enabled
+  if (loggingConfig.console) {
+    transports.push(new winston.transports.Console({
+      format: winston.format.combine(
+        winston.format.colorize(),
+        winston.format.simple()
+      )
+    }));
+  }
+
+  // Add file transports if logging is enabled
+  if (loggingConfig.enabled) {
+    const logDir = loggingConfig.logDirectory || path.join(process.cwd(), "zlogs");
+    
+    if (!fs.existsSync(logDir)) {
+      fs.mkdirSync(logDir, { recursive: true });
+    }
+
+    transports.push(
+      new winston.transports.File({
+        filename: path.join(logDir, "error.log"),
+        level: "error",
+      }),
+      new winston.transports.File({
+        filename: path.join(logDir, "combined.log"),
+      })
+    );
+  }
+
+  return winston.createLogger({
+    level: loggingConfig.level,
+    format: winston.format.combine(
+      winston.format.timestamp(),
+      winston.format.errors({ stack: true }),
+      winston.format.json()
+    ),
+    defaultMeta: { service: "@njdamstra/appwrite-utils-cli" },
+    transports,
+    silent: !loggingConfig.enabled && !loggingConfig.console,
+  });
+};
+
+export let logger = createLogger();
+
+// Recreate logger when config changes
+export const updateLogger = () => {
+  logger = createLogger();
+};
+
+/**
+ * Enable debug logging for troubleshooting push process issues
+ * This is a convenience function for quickly enabling comprehensive logging
+ */
+export const enableDebugLogging = (logDirectory?: string) => {
+  configureLogging({
+    enabled: true,
+    level: 'debug',
+    console: true,
+    logDirectory
+  });
+  updateLogger();
+  logger.info('Debug logging enabled for push process troubleshooting', {
+    level: 'debug',
+    console: true,
+    logDirectory: logDirectory || 'zlogs',
+    operation: 'enableDebugLogging'
+  });
+};
+
+/**
+ * Disable logging (reset to default)
+ */
+export const disableLogging = () => {
+  configureLogging(DEFAULT_LOGGING_CONFIG);
+  updateLogger();
+};
+
+/**
+ * Get current logging configuration
+ */
+export const getLoggingConfig = () => ({ ...loggingConfig });

package/src/shared/messageFormatter.ts

@@ -0,0 +1,208 @@
+import chalk from "chalk";
+import { logger } from "./logging.js";
+
+export interface MessageOptions {
+  prefix?: string;
+  skipLogging?: boolean;
+  logLevel?: "info" | "warn" | "error" | "debug";
+}
+
+export class MessageFormatter {
+  static success(message: string, options: MessageOptions = {}) {
+    const formatted = `${chalk.green("✅")} ${options.prefix ? `${options.prefix}: ` : ""}${message}`;
+    console.log(formatted);
+    
+    if (!options.skipLogging) {
+      logger.info(`SUCCESS: ${options.prefix ? `${options.prefix}: ` : ""}${message}`);
+    }
+  }
+
+  static error(message: string, error?: Error | string, options: MessageOptions = {}) {
+    const errorDetails = error instanceof Error ? error.message : error;
+    const formatted = `${chalk.red("❌")} ${options.prefix ? `${options.prefix}: ` : ""}${message}${errorDetails ? `\n   ${chalk.gray(errorDetails)}` : ""}`;
+    console.error(formatted);
+    
+    if (!options.skipLogging) {
+      logger.error(`ERROR: ${options.prefix ? `${options.prefix}: ` : ""}${message}`, {
+        error: errorDetails,
+        stack: error instanceof Error ? error.stack : undefined,
+      });
+    }
+  }
+
+  static warning(message: string, options: MessageOptions = {}) {
+    const formatted = `${chalk.yellow("⚠️")} ${options.prefix ? `${options.prefix}: ` : ""}${message}`;
+    console.log(formatted);
+    
+    if (!options.skipLogging) {
+      logger.warn(`WARNING: ${options.prefix ? `${options.prefix}: ` : ""}${message}`);
+    }
+  }
+
+  static info(message: string, options: MessageOptions = {}) {
+    const formatted = `${chalk.blue("ℹ️")} ${options.prefix ? `${options.prefix}: ` : ""}${message}`;
+    console.log(formatted);
+    
+    if (!options.skipLogging) {
+      logger.info(`INFO: ${options.prefix ? `${options.prefix}: ` : ""}${message}`);
+    }
+  }
+
+  static step(step: number, total: number, message: string, options: MessageOptions = {}) {
+    const formatted = `${chalk.cyan(`[${step}/${total}]`)} ${options.prefix ? `${options.prefix}: ` : ""}${message}`;
+    console.log(formatted);
+    
+    if (!options.skipLogging) {
+      logger.info(`STEP ${step}/${total}: ${options.prefix ? `${options.prefix}: ` : ""}${message}`);
+    }
+  }
+
+  static progress(message: string, options: MessageOptions = {}) {
+    const formatted = `${chalk.gray("⏳")} ${options.prefix ? `${options.prefix}: ` : ""}${message}`;
+    console.log(formatted);
+    
+    if (!options.skipLogging) {
+      logger.debug(`PROGRESS: ${options.prefix ? `${options.prefix}: ` : ""}${message}`);
+    }
+  }
+
+  static debug(message: string, data?: any, options: MessageOptions = {}) {
+    if (process.env.NODE_ENV === "development" || process.env.DEBUG) {
+      const formatted = `${chalk.magenta("🔍")} ${options.prefix ? `${options.prefix}: ` : ""}${message}`;
+      console.log(formatted);
+      if (data) {
+        console.log(chalk.gray(JSON.stringify(data, null, 2)));
+      }
+    }
+
+    if (!options.skipLogging) {
+      logger.debug(`DEBUG: ${options.prefix ? `${options.prefix}: ` : ""}${message}`, data);
+    }
+  }
+
+  static processing(message: string, options: MessageOptions = {}) {
+    const formatted = `${chalk.cyan("⚙️")} ${options.prefix ? `${options.prefix}: ` : ""}${message}`;
+    console.log(formatted);
+
+    if (!options.skipLogging) {
+      logger.info(`PROCESSING: ${options.prefix ? `${options.prefix}: ` : ""}${message}`);
+    }
+  }
+
+  static divider() {
+    console.log(chalk.gray("─".repeat(60)));
+  }
+
+  static banner(title: string, subtitle?: string) {
+    const divider = chalk.cyan("═".repeat(60));
+    console.log(`\n${divider}`);
+    console.log(chalk.cyan.bold(`  ${title}`));
+    if (subtitle) {
+      console.log(chalk.gray(`  ${subtitle}`));
+    }
+    console.log(`${divider}\n`);
+  }
+
+  static section(title: string) {
+    console.log(`\n${chalk.bold.underline(title)}`);
+  }
+
+  static list(items: string[], title?: string) {
+    if (title) {
+      console.log(chalk.bold(title));
+    }
+    items.forEach((item, index) => {
+      console.log(`${chalk.gray(`  ${index + 1}.`)} ${item}`);
+    });
+  }
+
+  static table(data: Record<string, string | number>[], headers?: string[]) {
+    if (data.length === 0) return;
+    
+    const keys = headers || Object.keys(data[0]);
+    const maxWidths = keys.map(key => 
+      Math.max(
+        key.length,
+        ...data.map(row => String(row[key]).length)
+      )
+    );
+
+    // Header
+    const headerRow = keys.map((key, i) => 
+      chalk.bold(key.padEnd(maxWidths[i]))
+    ).join(" │ ");
+    console.log(`┌─${keys.map((_, i) => "─".repeat(maxWidths[i])).join("─┼─")}─┐`);
+    console.log(`│ ${headerRow} │`);
+    console.log(`├─${keys.map((_, i) => "─".repeat(maxWidths[i])).join("─┼─")}─┤`);
+
+    // Rows
+    data.forEach(row => {
+      const dataRow = keys.map((key, i) => 
+        String(row[key]).padEnd(maxWidths[i])
+      ).join(" │ ");
+      console.log(`│ ${dataRow} │`);
+    });
+    console.log(`└─${keys.map((_, i) => "─".repeat(maxWidths[i])).join("─┴─")}─┘`);
+  }
+
+  static operationSummary(title: string, stats: Record<string, string | number>, duration?: number) {
+    this.section(`${title} Summary`);
+    
+    Object.entries(stats).forEach(([key, value]) => {
+      const formattedKey = key.replace(/([A-Z])/g, ' $1').replace(/^./, str => str.toUpperCase());
+      console.log(`${chalk.gray("●")} ${formattedKey}: ${chalk.cyan(String(value))}`);
+    });
+    
+    if (duration) {
+      console.log(`${chalk.gray("●")} Duration: ${chalk.cyan(this.formatDuration(duration))}`);
+    }
+    console.log();
+  }
+
+  static formatBytes(bytes: number): string {
+    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    let size = bytes;
+    let unitIndex = 0;
+    
+    while (size >= 1024 && unitIndex < units.length - 1) {
+      size /= 1024;
+      unitIndex++;
+    }
+    
+    return `${size.toFixed(1)} ${units[unitIndex]}`;
+  }
+
+  static formatDuration(ms: number): string {
+    const seconds = Math.floor(ms / 1000);
+    const minutes = Math.floor(seconds / 60);
+    const hours = Math.floor(minutes / 60);
+    
+    if (hours > 0) {
+      return `${hours}h ${minutes % 60}m ${seconds % 60}s`;
+    } else if (minutes > 0) {
+      return `${minutes}m ${seconds % 60}s`;
+    } else {
+      return `${seconds}s`;
+    }
+  }
+
+  static formatNumber(num: number): string {
+    return num.toLocaleString();
+  }
+}
+
+export const Messages = {
+  CONFIG_NOT_FOUND: "Appwrite configuration not found. Run 'appwrite-migrate setup' first.",
+  CONFIG_LOADED: (type: string, path: string) => `Loaded ${type} configuration from ${path}`,
+  DATABASE_CONNECTION_FAILED: "Failed to connect to Appwrite. Check your endpoint and API key.",
+  OPERATION_CANCELLED: "Operation cancelled by user.",
+  OPERATION_COMPLETED: (operation: string) => `${operation} completed successfully`,
+  BACKUP_STARTED: (database: string) => `Starting backup for database: ${database}`,
+  BACKUP_COMPLETED: (database: string, size: number) => `Backup completed for ${database} (${MessageFormatter.formatBytes(size)})`,
+  IMPORT_STARTED: (collections: number) => `Starting import process for ${collections} collection(s)`,
+  IMPORT_COMPLETED: (documents: number) => `Import completed. Processed ${MessageFormatter.formatNumber(documents)} documents`,
+  FUNCTION_DEPLOYED: (name: string) => `Function '${name}' deployed successfully`,
+  FUNCTION_DEPLOYMENT_FAILED: (name: string, error: string) => `Function '${name}' deployment failed: ${error}`,
+  TRANSFER_STARTED: (source: string, target: string) => `Starting transfer from ${source} to ${target}`,
+  TRANSFER_COMPLETED: (items: number) => `Transfer completed. Moved ${MessageFormatter.formatNumber(items)} items`,
+} as const;

package/src/shared/migrationHelpers.ts

@@ -0,0 +1,232 @@
+import { ID, Query, type Databases } from "node-appwrite";
+import { BatchSchema, OperationSchema, type Operation } from "../storage/schemas.js";
+import { AttributeMappingsSchema } from "@njdamstra/appwrite-utils";
+import { z } from "zod";
+import { logger } from "./logging.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import {
+  findOrCreateOperation as findOrCreateOp,
+  updateOperation as updateOp,
+  getOperation as getOp
+} from "./operationsTable.js";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
+import { MessageFormatter } from "./messageFormatter.js";
+
+/**
+ * Object that contains the context for an action that needs to be executed after import
+ * Used in the afterImportActionsDefinitions
+ * @type {ContextObject}
+ * @typedef {Object} ContextObject
+ * @property {string} collectionId - The ID of the collection
+ * @property {any} finalItem - The final item that was imported
+ * @property {string} action - The name of the action
+ * @property {string[]} params - The parameters for the action
+ * @property {Object} context - The context object for the action (all the data of this specific item)
+ */
+export const ContextObject = z.object({
+  dbId: z.string(),
+  collectionId: z.string(),
+  finalItem: z.any(),
+  attributeMappings: AttributeMappingsSchema,
+  context: z.any(),
+});
+
+export type ContextObject = z.infer<typeof ContextObject>;
+
+export const createOrFindAfterImportOperation = async (
+  database: Databases,
+  collectionId: string,
+  context: ContextObject
+) => {
+  let operation = await findOrCreateOperationLegacy(
+    database,
+    collectionId,
+    "afterImportAction"
+  );
+  if (!operation.batches) {
+    operation.batches = [];
+  }
+
+  // Directly create a new batch for the context without checking for an existing batch
+  const contextData = JSON.stringify(context);
+  // Create a new batch with the contextData
+  const newBatchId = await addBatch(database, contextData);
+  // Update the operation with the new batch's $id
+  operation.batches = [...operation.batches, newBatchId];
+  await database.updateDocument(
+    "migrations",
+    "currentOperations",
+    operation.$id,
+    { batches: operation.batches }
+  );
+};
+
+export const addBatch = async (database: Databases, data: string) => {
+  const batch = await database.createDocument(
+    "migrations",
+    "batches",
+    ID.unique(),
+    {
+      data,
+      processed: false,
+    }
+  );
+  return batch.$id;
+};
+
+export const getAfterImportOperations = async (
+  database: Databases,
+  collectionId: string
+) => {
+  let lastDocumentId: string | undefined;
+  const allOperations = [];
+  let total = 0;
+
+  do {
+    const query = [
+      Query.equal("collectionId", collectionId),
+      Query.equal("operationType", "afterImportAction"),
+      Query.limit(100),
+    ];
+
+    if (lastDocumentId) {
+      query.push(Query.cursorAfter(lastDocumentId));
+    }
+
+    const operations = await database.listDocuments(
+      "migrations",
+      "currentOperations",
+      query
+    );
+    total = operations.total; // Update total with the latest fetch
+    allOperations.push(...operations.documents);
+
+    if (operations.documents.length > 0 && operations.documents.length >= 100) {
+      lastDocumentId =
+        operations.documents[operations.documents.length - 1].$id;
+    }
+  } while (allOperations.length < total);
+
+  const allOps = allOperations.map((op) => OperationSchema.parse(op));
+  return allOps;
+};
+
+// Legacy function for backward compatibility with old migrations database
+const findOrCreateOperationLegacy = async (
+  database: Databases,
+  collectionId: string,
+  operationType: string,
+  additionalQueries?: string[]
+) => {
+  const operations = await tryAwaitWithRetry(
+    async () =>
+      await database.listDocuments("migrations", "currentOperations", [
+        Query.equal("collectionId", collectionId),
+        Query.equal("operationType", operationType),
+        Query.equal("status", "pending"),
+        ...(additionalQueries || []),
+      ])
+  );
+
+  if (operations.documents.length > 0) {
+    return OperationSchema.parse(operations.documents[0]);
+  } else {
+    const op = await tryAwaitWithRetry(
+      async () =>
+        await database.createDocument(
+          "migrations",
+          "currentOperations",
+          ID.unique(),
+          {
+            operationType,
+            collectionId,
+            status: "pending",
+            batches: [],
+            progress: 0,
+            total: 0,
+            error: "",
+          }
+        )
+    );
+
+    return OperationSchema.parse(op);
+  }
+};
+
+export const findOrCreateOperation = async (
+  db: DatabaseAdapter,
+  databaseId: string,
+  operationType: string,
+  collectionId?: string,
+  data?: any
+): Promise<any> => {
+  // Use new operations table system
+  return await findOrCreateOp(db, databaseId, operationType, {
+    targetCollection: collectionId,
+    data: data
+  });
+};
+
+export const updateOperation = async (
+  db: DatabaseAdapter,
+  databaseId: string,
+  operationId: string,
+  updates: any
+): Promise<any> => {
+  // Use new operations table system
+  return await updateOp(db, databaseId, operationId, updates);
+};
+
+export const getOperation = async (
+  db: DatabaseAdapter,
+  databaseId: string,
+  operationId: string
+): Promise<any> => {
+  // Use new operations table system
+  return await getOp(db, databaseId, operationId);
+};
+
+// Actual max 1073741824
+export const maxDataLength = 1073741820;
+export const maxBatchItems = 25;
+
+export const splitIntoBatches = (data: any[]): any[][] => {
+  let batches = [];
+  let currentBatch: any[] = [];
+  let currentBatchLength = 0;
+  let currentBatchItemCount = 0;
+
+  data.forEach((item, index) => {
+    const itemLength = JSON.stringify(item).length;
+    if (itemLength > maxDataLength) {
+      MessageFormatter.warning(
+        `Large item found at index ${index} with length ${itemLength}`,
+        { prefix: "Batch Splitter" }
+      );
+      logger.debug("Large item data:", item);
+    }
+    // Check if adding the current item would exceed the max length or max items per batch
+    if (
+      currentBatchLength + itemLength >= maxDataLength ||
+      currentBatchItemCount >= maxBatchItems
+    ) {
+      // If so, start a new batch
+      batches.push(currentBatch);
+      currentBatch = [item];
+      currentBatchLength = itemLength;
+      currentBatchItemCount = 1; // Reset item count for the new batch
+    } else {
+      // Otherwise, add the item to the current batch
+      currentBatch.push(item);
+      currentBatchLength += itemLength;
+      currentBatchItemCount++;
+    }
+  });
+
+  // Don't forget to add the last batch if it's not empty
+  if (currentBatch.length > 0) {
+    batches.push(currentBatch);
+  }
+
+  return batches;
+};

package/src/shared/operationLogger.ts

@@ -0,0 +1,20 @@
+import type { Databases, Models } from "node-appwrite";
+import type { OperationCreate } from "../storage/schemas.js";
+
+/**
+ * Legacy operation logger - deprecated
+ * This function is maintained for backward compatibility but no longer performs any logging.
+ * The operations table system has been refactored to use the dynamic adapter pattern.
+ *
+ * @deprecated This function will be removed in a future version
+ */
+export const logOperation = async (
+  db: Databases,
+  dbId: string,
+  operationDetails: OperationCreate,
+  operationId?: string
+): Promise<Models.Document | null> => {
+  // No-op: Operations logging has been moved to the new operations table system
+  // Callers should migrate to using operationsTable.ts functions directly
+  return null;
+};