@njdamstra/appwrite-utils-cli 1.8.9

package/src/shared/operationQueue.ts

@@ -0,0 +1,377 @@
+import { Query, type Databases, type Models } from "node-appwrite";
+import type { Attribute } from "@njdamstra/appwrite-utils";
+import { createOrUpdateAttributeWithStatusCheck } from "../collections/attributes.js";
+import { fetchAndCacheCollectionByName } from "../collections/methods.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
+import { logger } from "../shared/logging.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+
+export interface QueuedOperation {
+  type: "attribute";
+  collectionId?: string;
+  attribute?: Attribute;
+  collection?: Models.Collection;
+  dependencies?: string[];
+}
+
+// Global state management
+export const queuedOperations: QueuedOperation[] = [];
+export const nameToIdMapping: Map<string, string> = new Map();
+// Keys are scoped per database to avoid cross-database collisions
+// Collections key format: `${databaseId}::${collectionId}`
+// Attributes key format: `${databaseId}::${collectionId}::${attributeKey}`
+export const processedCollections: Set<string> = new Set();
+export const processedAttributes: Set<string> = new Set();
+
+// Helpers to build scoped keys
+const collectionKey = (databaseId: string, collectionId: string) => `${databaseId}::${collectionId}`;
+const attributeKeyScoped = (databaseId: string, collectionId: string, key: string) => `${databaseId}::${collectionId}::${key}`;
+
+export const enqueueOperation = (operation: QueuedOperation) => {
+  // Avoid duplicate queue entries for same attribute
+  const attributeKey = operation.attribute?.key;
+  const collectionId = operation.collectionId;
+
+  logger.info('Enqueueing operation', {
+    type: operation.type,
+    attributeKey,
+    collectionId,
+    dependencies: operation.dependencies,
+    queueSizeBefore: queuedOperations.length,
+    operation: 'enqueueOperation'
+  });
+
+  if (attributeKey && collectionId) {
+    const duplicateIndex = queuedOperations.findIndex(
+      (op) => op.collectionId === collectionId && op.attribute?.key === attributeKey
+    );
+
+    if (duplicateIndex !== -1) {
+      MessageFormatter.info(`Replacing existing queue entry for attribute: ${attributeKey}`);
+      logger.info('Replacing duplicate queue entry', {
+        attributeKey,
+        collectionId,
+        duplicateIndex,
+        operation: 'enqueueOperation'
+      });
+      queuedOperations[duplicateIndex] = operation;
+      return;
+    }
+  }
+
+  queuedOperations.push(operation);
+  logger.debug('Operation enqueued successfully', {
+    attributeKey,
+    collectionId,
+    queueSizeAfter: queuedOperations.length,
+    operation: 'enqueueOperation'
+  });
+};
+
+/**
+ * Clear all caches and processing state - use between operations
+ */
+export const clearProcessingState = () => {
+  const sizeBefore = {
+    collections: processedCollections.size,
+    attributes: processedAttributes.size,
+    nameMapping: nameToIdMapping.size
+  };
+
+  processedCollections.clear();
+  processedAttributes.clear();
+  nameToIdMapping.clear();
+
+  logger.debug("Cleared processing state caches", { operation: "clearProcessingState", sizeBefore });
+  logger.info('Processing state cleared', {
+    sizeBefore,
+    operation: 'clearProcessingState'
+  });
+};
+
+/**
+ * Check if a collection has already been fully processed
+ */
+export const isCollectionProcessed = (collectionId: string, databaseId: string): boolean => {
+  return processedCollections.has(collectionKey(databaseId, collectionId));
+};
+
+/**
+ * Mark a collection as fully processed
+ */
+export const markCollectionProcessed = (collectionId: string, collectionName: string | undefined, databaseId: string) => {
+  processedCollections.add(collectionKey(databaseId, collectionId));
+
+  const logData = {
+    databaseId,
+    collectionId,
+    collectionName,
+    totalProcessedCollections: processedCollections.size,
+    operation: 'markCollectionProcessed'
+  };
+
+  if (collectionName) {
+    MessageFormatter.success(`Marked collection '${collectionName}' (${collectionId}) as processed`, { prefix: 'Tables' });
+  }
+
+  logger.info('Collection marked as processed', logData);
+};
+
+/**
+ * Check if a specific attribute has been processed
+ */
+export const isAttributeProcessed = (databaseId: string, collectionId: string, attributeKey: string): boolean => {
+  return processedAttributes.has(attributeKeyScoped(databaseId, collectionId, attributeKey));
+};
+
+/**
+ * Mark a specific attribute as processed
+ */
+export const markAttributeProcessed = (databaseId: string, collectionId: string, attributeKey: string) => {
+  const identifier = attributeKeyScoped(databaseId, collectionId, attributeKey);
+  processedAttributes.add(identifier);
+
+  logger.debug('Attribute marked as processed', {
+    databaseId,
+    collectionId,
+    attributeKey,
+    identifier,
+    totalProcessedAttributes: processedAttributes.size,
+    operation: 'markAttributeProcessed'
+  });
+};
+
+/**
+ * Process only specific attributes in the queue, not entire collections
+ * This prevents triggering full collection re-processing cycles
+ */
+export const processQueue = async (db: Databases | DatabaseAdapter, dbId: string) => {
+  const startTime = Date.now();
+
+  if (queuedOperations.length === 0) {
+    MessageFormatter.info("No queued operations to process");
+    logger.info('Queue processing skipped - no operations', {
+      dbId,
+      operation: 'processQueue'
+    });
+    return;
+  }
+
+  MessageFormatter.section(`Starting surgical queue processing of ${queuedOperations.length} operations for ${dbId}`);
+
+  logger.info('Starting queue processing', {
+    dbId,
+    queueSize: queuedOperations.length,
+    operations: queuedOperations.map(op => ({
+      type: op.type,
+      attributeKey: op.attribute?.key,
+      collectionId: op.collectionId,
+      dependencies: op.dependencies
+    })),
+    operation: 'processQueue'
+  });
+
+  let progress = true;
+  let attempts = 0;
+  const maxAttempts = 3; // Prevent infinite loops
+
+  while (progress && attempts < maxAttempts) {
+    progress = false;
+    attempts++;
+
+    MessageFormatter.info(`Queue processing attempt ${attempts}/${maxAttempts}`);
+
+    logger.info('Queue processing attempt started', {
+      attempt: attempts,
+      maxAttempts,
+      remainingOperations: queuedOperations.length,
+      dbId,
+      operation: 'processQueue'
+    });
+
+    for (let i = queuedOperations.length - 1; i >= 0; i--) {
+      const operation = queuedOperations[i];
+
+      if (!operation.attribute || !operation.collectionId) {
+        MessageFormatter.warning("Invalid operation, removing from queue");
+        queuedOperations.splice(i, 1);
+        continue;
+      }
+
+      const attributeKey = operation.attribute.key;
+      const collectionId = operation.collectionId;
+
+      // Skip if this specific attribute was already processed (per database)
+      if (isAttributeProcessed(dbId, collectionId, attributeKey)) {
+        MessageFormatter.debug(`Attribute '${attributeKey}' already processed, removing from queue`);
+        logger.debug('Removing already processed attribute from queue', {
+          attributeKey,
+          collectionId,
+          queueIndex: i,
+          operation: 'processQueue'
+        });
+        queuedOperations.splice(i, 1);
+        continue;
+      }
+
+      let targetCollection: Models.Collection | undefined;
+
+      // Resolve the target collection (where the attribute will be created)
+      try {
+        targetCollection = await tryAwaitWithRetry(
+          async () => {
+            if ('getMetadata' in db && typeof db.getMetadata === 'function') {
+              // DatabaseAdapter
+              return (await (db as DatabaseAdapter).getTable({ databaseId: dbId, tableId: collectionId })).data;
+            } else {
+              // Legacy Databases
+              return await (db as Databases).getCollection(dbId, collectionId);
+            }
+          }
+        );
+      } catch (e) {
+        const errorMessage = e instanceof Error ? e.message : String(e);
+        MessageFormatter.error(`Target collection ${collectionId} not found, removing from queue`);
+        logger.error('Target collection not found during queue processing', {
+          collectionId,
+          attributeKey,
+          error: errorMessage,
+          operation: 'processQueue'
+        });
+        queuedOperations.splice(i, 1);
+        continue;
+      }
+
+      // For relationship attributes, ensure the related collection exists
+      let canProcess = true;
+      if (operation.attribute.type === "relationship") {
+        const relatedCollection = operation.attribute.relatedCollection;
+        if (relatedCollection) {
+          // Try to resolve related collection by ID first, then by name
+          let relatedFound = false;
+
+          try {
+            await tryAwaitWithRetry(
+              async () => {
+                if ('getMetadata' in db && typeof db.getMetadata === 'function') {
+                  // DatabaseAdapter
+                  return (await (db as DatabaseAdapter).getTable({ databaseId: dbId, tableId: relatedCollection })).data;
+                } else {
+                  // Legacy Databases
+                  return await (db as Databases).getCollection(dbId, relatedCollection);
+                }
+              }
+            );
+            relatedFound = true;
+            nameToIdMapping.set(relatedCollection, relatedCollection); // Cache the ID mapping
+          } catch (_) {
+            // Try by name lookup
+            const cachedId = nameToIdMapping.get(relatedCollection);
+            if (cachedId) {
+              try {
+                await tryAwaitWithRetry(
+                  async () => {
+                    if ('getMetadata' in db && typeof db.getMetadata === 'function') {
+                      // DatabaseAdapter
+                      return (await (db as DatabaseAdapter).getTable({ databaseId: dbId, tableId: cachedId })).data;
+                    } else {
+                      // Legacy Databases
+                      return await (db as Databases).getCollection(dbId, cachedId);
+                    }
+                  }
+                );
+                relatedFound = true;
+              } catch (_) {
+                nameToIdMapping.delete(relatedCollection); // Remove stale cache
+              }
+            }
+
+            if (!relatedFound) {
+              // Final attempt: search by name
+              try {
+                const collections = 'getMetadata' in db && typeof db.getMetadata === 'function'
+                  ? await (db as DatabaseAdapter).listTables({ databaseId: dbId, queries: [Query.equal("name", relatedCollection)] })
+                  : await (db as Databases).listCollections(dbId, [Query.equal("name", relatedCollection)]);
+
+                if (collections.total && collections.total > 0) {
+                  const firstCollection = 'getMetadata' in db && typeof db.getMetadata === 'function'
+                    ? (collections as any).tables?.[0]
+                    : (collections as any).collections?.[0];
+                  nameToIdMapping.set(relatedCollection, firstCollection.$id);
+                  relatedFound = true;
+                }
+              } catch (_) {
+                // Related collection truly doesn't exist yet
+              }
+            }
+          }
+
+          if (!relatedFound) {
+            MessageFormatter.warning(
+              `Related collection '${relatedCollection}' not ready for attribute '${attributeKey}', keeping in queue`
+            );
+            canProcess = false;
+          }
+        }
+      }
+
+      if (canProcess && targetCollection) {
+        MessageFormatter.progress(
+          `Processing queued ${operation.attribute.type} attribute: '${attributeKey}' for collection: '${targetCollection.name}'`
+        );
+
+        const success = await createOrUpdateAttributeWithStatusCheck(
+          db,
+          dbId,
+          targetCollection,
+          operation.attribute
+        );
+
+        if (success) {
+          MessageFormatter.success(`Successfully processed queued attribute: '${attributeKey}'`);
+          logger.info('Queued attribute processed successfully', {
+            attributeKey,
+            collectionId,
+            targetCollectionName: targetCollection.name,
+            operation: 'processQueue'
+          });
+          markAttributeProcessed(dbId, collectionId, attributeKey);
+          queuedOperations.splice(i, 1);
+          progress = true;
+        } else {
+          MessageFormatter.error(`Failed to process queued attribute: '${attributeKey}', removing from queue`);
+          logger.error('Failed to process queued attribute', {
+            attributeKey,
+            collectionId,
+            targetCollectionName: targetCollection.name,
+            operation: 'processQueue'
+          });
+          queuedOperations.splice(i, 1);
+        }
+      }
+    }
+
+    if (queuedOperations.length === 0) {
+      break;
+    }
+
+    MessageFormatter.info(`Remaining operations after attempt ${attempts}: ${queuedOperations.length}`);
+  }
+
+  if (queuedOperations.length > 0) {
+    MessageFormatter.warning(
+      `${queuedOperations.length} operations remain unresolved after ${maxAttempts} attempts:`
+    );
+    queuedOperations.forEach((op, index) => {
+      MessageFormatter.warning(
+        `  ${index + 1}. ${op.attribute?.type} attribute '${op.attribute?.key}' for collection ${op.collectionId}`
+      );
+    });
+    MessageFormatter.warning("These may have unmet dependencies or require manual intervention");
+  } else {
+    MessageFormatter.success("All queued operations processed successfully");
+  }
+
+  MessageFormatter.section(`Surgical queue processing complete for ${dbId}`);
+};

package/src/shared/operationsTable.ts

@@ -0,0 +1,338 @@
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
+import { logger } from "./logging.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { Query, ID } from "node-appwrite";
+import {
+	OPERATIONS_TABLE_ID,
+	OPERATIONS_TABLE_NAME,
+	type OperationRecord,
+	type OperationStatus,
+	OperationRecordSchema,
+} from "./operationsTableSchema.js";
+
+/**
+ * Operations Table Manager
+ *
+ * Manages dynamic operations tracking tables in each database.
+ * Creates _appwrite_operations table on-demand for tracking imports, exports, transfers, etc.
+ */
+
+/**
+ * Checks if operations table exists in database
+ */
+async function tableExists(
+	db: DatabaseAdapter,
+	databaseId: string,
+): Promise<boolean> {
+	try {
+		await db.getTable({ databaseId, tableId: OPERATIONS_TABLE_ID });
+		return true;
+	} catch (error) {
+		return false;
+	}
+}
+
+/**
+ * Creates the operations tracking table in the specified database
+ * Table is created with underscore prefix to indicate it's a system table
+ */
+export async function createOperationsTable(
+	db: DatabaseAdapter,
+	databaseId: string,
+): Promise<void> {
+	// Check if table already exists
+	const exists = await tableExists(db, databaseId);
+	if (exists) {
+		logger.debug("Operations table already exists", {
+			databaseId,
+			tableId: OPERATIONS_TABLE_ID,
+		});
+		return;
+	}
+
+	logger.info("Creating operations tracking table", {
+		databaseId,
+		tableId: OPERATIONS_TABLE_ID,
+	});
+
+	// Create table
+	await tryAwaitWithRetry(async () => {
+		await db.createTable({
+			databaseId,
+			id: OPERATIONS_TABLE_ID,
+			name: OPERATIONS_TABLE_NAME,
+		});
+	});
+
+	// Create attributes with retry logic
+	const attributes = [
+		{
+			key: "operationType",
+			type: "string",
+			size: 50,
+			required: true,
+		},
+		{
+			key: "targetCollection",
+			type: "string",
+			size: 50,
+			required: false,
+		},
+		{
+			key: "status",
+			type: "enum",
+			elements: ["pending", "in_progress", "completed", "failed", "cancelled"],
+			required: true,
+			default: "pending",
+		},
+		{
+			key: "progress",
+			type: "integer",
+			required: true,
+			default: 0,
+		},
+		{
+			key: "total",
+			type: "integer",
+			required: true,
+			default: 0,
+		},
+		{
+			key: "data",
+			type: "string",
+			size: 1048576, // 1MB for serialized data
+			required: false,
+		},
+		{
+			key: "error",
+			type: "string",
+			size: 10000,
+			required: false,
+		},
+	];
+
+	for (const attr of attributes) {
+		await tryAwaitWithRetry(async () => {
+			await db.createAttribute({
+				databaseId,
+				tableId: OPERATIONS_TABLE_ID,
+				...attr,
+			});
+		});
+	}
+
+	logger.info("Operations tracking table created successfully", {
+		databaseId,
+		tableId: OPERATIONS_TABLE_ID,
+		attributes: attributes.length,
+	});
+}
+
+/**
+ * Finds an existing operation or creates a new one
+ * Useful for resuming interrupted operations
+ */
+export async function findOrCreateOperation(
+	db: DatabaseAdapter,
+	databaseId: string,
+	operationType: string,
+	params?: Partial<OperationRecord>,
+): Promise<OperationRecord> {
+	// Ensure operations table exists
+	await createOperationsTable(db, databaseId);
+
+	// Try to find existing pending operation
+	try {
+		const queries = [
+			Query.equal("operationType", operationType),
+			Query.equal("status", "pending"),
+		];
+
+		if (params?.targetCollection) {
+			queries.push(Query.equal("targetCollection", params.targetCollection));
+		}
+
+		const response = await db.listRows({
+			databaseId,
+			tableId: OPERATIONS_TABLE_ID,
+			queries,
+		});
+
+		if (response.rows && response.rows.length > 0) {
+			logger.debug("Found existing operation", {
+				operationId: response.rows[0].$id,
+				operationType,
+			});
+			return response.rows[0] as OperationRecord;
+		}
+	} catch (error) {
+		logger.debug("No existing operation found, creating new one", {
+			operationType,
+			error: error instanceof Error ? error.message : String(error),
+		});
+	}
+
+	// Create new operation
+	const newOperation = {
+		operationType,
+		targetCollection: params?.targetCollection,
+		status: "pending" as OperationStatus,
+		progress: params?.progress ?? 0,
+		total: params?.total ?? 0,
+		data: params?.data ? JSON.stringify(params.data) : undefined,
+		error: params?.error,
+	};
+
+	const result = await db.createRow({
+		databaseId,
+		tableId: OPERATIONS_TABLE_ID,
+		id: ID.unique(),
+		data: newOperation,
+	});
+
+	logger.info("Created new operation", {
+		operationId: result.data?.$id,
+		operationType,
+	});
+
+	return result.data as OperationRecord;
+}
+
+/**
+ * Updates an existing operation record
+ */
+export async function updateOperation(
+	db: DatabaseAdapter,
+	databaseId: string,
+	operationId: string,
+	updates: Partial<OperationRecord>,
+): Promise<OperationRecord> {
+	// Prepare update data (exclude system fields like $id, $createdAt, $updatedAt)
+	const updateData: Record<string, any> = {};
+
+	if (updates.operationType !== undefined)
+		updateData.operationType = updates.operationType;
+	if (updates.targetCollection !== undefined)
+		updateData.targetCollection = updates.targetCollection;
+	if (updates.status !== undefined) updateData.status = updates.status;
+	if (updates.progress !== undefined) updateData.progress = updates.progress;
+	if (updates.total !== undefined) updateData.total = updates.total;
+	if (updates.error !== undefined) updateData.error = updates.error;
+	if (updates.data !== undefined) {
+		updateData.data =
+			typeof updates.data === "string"
+				? updates.data
+				: JSON.stringify(updates.data);
+	}
+
+	try {
+		const result = await db.updateRow({
+			databaseId,
+			tableId: OPERATIONS_TABLE_ID,
+			id: operationId,
+			data: updateData,
+		});
+
+		logger.debug("Updated operation", {
+			operationId,
+			updates: Object.keys(updateData),
+		});
+
+		return result.data as OperationRecord;
+	} catch (error) {
+		logger.error("Failed to update operation", {
+			operationId,
+			error: error instanceof Error ? error.message : String(error),
+		});
+		throw error;
+	}
+}
+
+/**
+ * Gets a single operation by ID
+ */
+export async function getOperation(
+	db: DatabaseAdapter,
+	databaseId: string,
+	operationId: string,
+): Promise<OperationRecord | null> {
+	try {
+		const result = await db.getRow({
+			databaseId,
+			tableId: OPERATIONS_TABLE_ID,
+			id: operationId,
+		});
+
+		return result.data as OperationRecord;
+	} catch (error) {
+		logger.debug("Operation not found", {
+			operationId,
+			error: error instanceof Error ? error.message : String(error),
+		});
+		return null;
+	}
+}
+
+/**
+ * Cleans up old completed operations
+ * @param olderThan - Optional date, defaults to operations older than 7 days
+ * @returns Number of operations deleted
+ */
+export async function cleanupOperations(
+	db: DatabaseAdapter,
+	databaseId: string,
+	olderThan?: Date,
+): Promise<number> {
+	const cutoffDate =
+		olderThan || new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); // 7 days ago
+	const cutoffIso = cutoffDate.toISOString();
+
+	try {
+		// Query for completed operations older than cutoff
+		const response = await db.listRows({
+			databaseId,
+			tableId: OPERATIONS_TABLE_ID,
+			queries: [
+				Query.equal("status", ["completed", "failed", "cancelled"]),
+				Query.lessThan("$createdAt", cutoffIso),
+			],
+		});
+
+		if (!response.rows || response.rows.length === 0) {
+			logger.debug("No old operations to clean up", { databaseId });
+			return 0;
+		}
+
+		// Delete in batches
+		let deletedCount = 0;
+		for (const operation of response.rows) {
+			try {
+				await db.deleteRow({
+					databaseId,
+					tableId: OPERATIONS_TABLE_ID,
+					id: operation.$id,
+				});
+				deletedCount++;
+			} catch (error) {
+				logger.warn("Failed to delete operation", {
+					operationId: operation.$id,
+					error: error instanceof Error ? error.message : String(error),
+				});
+			}
+		}
+
+		logger.info("Cleaned up old operations", {
+			databaseId,
+			deletedCount,
+			cutoffDate: cutoffIso,
+		});
+
+		return deletedCount;
+	} catch (error) {
+		logger.error("Failed to cleanup operations", {
+			databaseId,
+			error: error instanceof Error ? error.message : String(error),
+		});
+		return 0;
+	}
+}