appwrite-utils-cli 1.9.7 → 1.12.0

package/src/shared/operationQueue.ts

@@ -1,377 +0,0 @@
-import { Query, type Databases, type Models } from "node-appwrite";
-import type { Attribute } from "appwrite-utils";
-import { createOrUpdateAttributeWithStatusCheck } from "../collections/attributes.js";
-import { fetchAndCacheCollectionByName } from "../collections/methods.js";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
-import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
-import { logger } from "../shared/logging.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
-
-export interface QueuedOperation {
-  type: "attribute";
-  collectionId?: string;
-  attribute?: Attribute;
-  collection?: Models.Collection;
-  dependencies?: string[];
-}
-
-// Global state management
-export const queuedOperations: QueuedOperation[] = [];
-export const nameToIdMapping: Map<string, string> = new Map();
-// Keys are scoped per database to avoid cross-database collisions
-// Collections key format: `${databaseId}::${collectionId}`
-// Attributes key format: `${databaseId}::${collectionId}::${attributeKey}`
-export const processedCollections: Set<string> = new Set();
-export const processedAttributes: Set<string> = new Set();
-
-// Helpers to build scoped keys
-const collectionKey = (databaseId: string, collectionId: string) => `${databaseId}::${collectionId}`;
-const attributeKeyScoped = (databaseId: string, collectionId: string, key: string) => `${databaseId}::${collectionId}::${key}`;
-
-export const enqueueOperation = (operation: QueuedOperation) => {
-  // Avoid duplicate queue entries for same attribute
-  const attributeKey = operation.attribute?.key;
-  const collectionId = operation.collectionId;
-
-  logger.info('Enqueueing operation', {
-    type: operation.type,
-    attributeKey,
-    collectionId,
-    dependencies: operation.dependencies,
-    queueSizeBefore: queuedOperations.length,
-    operation: 'enqueueOperation'
-  });
-
-  if (attributeKey && collectionId) {
-    const duplicateIndex = queuedOperations.findIndex(
-      (op) => op.collectionId === collectionId && op.attribute?.key === attributeKey
-    );
-
-    if (duplicateIndex !== -1) {
-      MessageFormatter.info(`Replacing existing queue entry for attribute: ${attributeKey}`);
-      logger.info('Replacing duplicate queue entry', {
-        attributeKey,
-        collectionId,
-        duplicateIndex,
-        operation: 'enqueueOperation'
-      });
-      queuedOperations[duplicateIndex] = operation;
-      return;
-    }
-  }
-
-  queuedOperations.push(operation);
-  logger.debug('Operation enqueued successfully', {
-    attributeKey,
-    collectionId,
-    queueSizeAfter: queuedOperations.length,
-    operation: 'enqueueOperation'
-  });
-};
-
-/**
- * Clear all caches and processing state - use between operations
- */
-export const clearProcessingState = () => {
-  const sizeBefore = {
-    collections: processedCollections.size,
-    attributes: processedAttributes.size,
-    nameMapping: nameToIdMapping.size
-  };
-
-  processedCollections.clear();
-  processedAttributes.clear();
-  nameToIdMapping.clear();
-
-  logger.debug("Cleared processing state caches", { operation: "clearProcessingState", sizeBefore });
-  logger.info('Processing state cleared', {
-    sizeBefore,
-    operation: 'clearProcessingState'
-  });
-};
-
-/**
- * Check if a collection has already been fully processed
- */
-export const isCollectionProcessed = (collectionId: string, databaseId: string): boolean => {
-  return processedCollections.has(collectionKey(databaseId, collectionId));
-};
-
-/**
- * Mark a collection as fully processed
- */
-export const markCollectionProcessed = (collectionId: string, collectionName: string | undefined, databaseId: string) => {
-  processedCollections.add(collectionKey(databaseId, collectionId));
-
-  const logData = {
-    databaseId,
-    collectionId,
-    collectionName,
-    totalProcessedCollections: processedCollections.size,
-    operation: 'markCollectionProcessed'
-  };
-
-  if (collectionName) {
-    MessageFormatter.success(`Marked collection '${collectionName}' (${collectionId}) as processed`, { prefix: 'Tables' });
-  }
-
-  logger.info('Collection marked as processed', logData);
-};
-
-/**
- * Check if a specific attribute has been processed
- */
-export const isAttributeProcessed = (databaseId: string, collectionId: string, attributeKey: string): boolean => {
-  return processedAttributes.has(attributeKeyScoped(databaseId, collectionId, attributeKey));
-};
-
-/**
- * Mark a specific attribute as processed
- */
-export const markAttributeProcessed = (databaseId: string, collectionId: string, attributeKey: string) => {
-  const identifier = attributeKeyScoped(databaseId, collectionId, attributeKey);
-  processedAttributes.add(identifier);
-
-  logger.debug('Attribute marked as processed', {
-    databaseId,
-    collectionId,
-    attributeKey,
-    identifier,
-    totalProcessedAttributes: processedAttributes.size,
-    operation: 'markAttributeProcessed'
-  });
-};
-
-/**
- * Process only specific attributes in the queue, not entire collections
- * This prevents triggering full collection re-processing cycles
- */
-export const processQueue = async (db: Databases | DatabaseAdapter, dbId: string) => {
-  const startTime = Date.now();
-
-  if (queuedOperations.length === 0) {
-    MessageFormatter.info("No queued operations to process");
-    logger.info('Queue processing skipped - no operations', {
-      dbId,
-      operation: 'processQueue'
-    });
-    return;
-  }
-
-  MessageFormatter.section(`Starting surgical queue processing of ${queuedOperations.length} operations for ${dbId}`);
-
-  logger.info('Starting queue processing', {
-    dbId,
-    queueSize: queuedOperations.length,
-    operations: queuedOperations.map(op => ({
-      type: op.type,
-      attributeKey: op.attribute?.key,
-      collectionId: op.collectionId,
-      dependencies: op.dependencies
-    })),
-    operation: 'processQueue'
-  });
-
-  let progress = true;
-  let attempts = 0;
-  const maxAttempts = 3; // Prevent infinite loops
-
-  while (progress && attempts < maxAttempts) {
-    progress = false;
-    attempts++;
-
-    MessageFormatter.info(`Queue processing attempt ${attempts}/${maxAttempts}`);
-
-    logger.info('Queue processing attempt started', {
-      attempt: attempts,
-      maxAttempts,
-      remainingOperations: queuedOperations.length,
-      dbId,
-      operation: 'processQueue'
-    });
-
-    for (let i = queuedOperations.length - 1; i >= 0; i--) {
-      const operation = queuedOperations[i];
-
-      if (!operation.attribute || !operation.collectionId) {
-        MessageFormatter.warning("Invalid operation, removing from queue");
-        queuedOperations.splice(i, 1);
-        continue;
-      }
-
-      const attributeKey = operation.attribute.key;
-      const collectionId = operation.collectionId;
-
-      // Skip if this specific attribute was already processed (per database)
-      if (isAttributeProcessed(dbId, collectionId, attributeKey)) {
-        MessageFormatter.debug(`Attribute '${attributeKey}' already processed, removing from queue`);
-        logger.debug('Removing already processed attribute from queue', {
-          attributeKey,
-          collectionId,
-          queueIndex: i,
-          operation: 'processQueue'
-        });
-        queuedOperations.splice(i, 1);
-        continue;
-      }
-
-      let targetCollection: Models.Collection | undefined;
-
-      // Resolve the target collection (where the attribute will be created)
-      try {
-        targetCollection = await tryAwaitWithRetry(
-          async () => {
-            if ('getMetadata' in db && typeof db.getMetadata === 'function') {
-              // DatabaseAdapter
-              return (await (db as DatabaseAdapter).getTable({ databaseId: dbId, tableId: collectionId })).data;
-            } else {
-              // Legacy Databases
-              return await (db as Databases).getCollection(dbId, collectionId);
-            }
-          }
-        );
-      } catch (e) {
-        const errorMessage = e instanceof Error ? e.message : String(e);
-        MessageFormatter.error(`Target collection ${collectionId} not found, removing from queue`);
-        logger.error('Target collection not found during queue processing', {
-          collectionId,
-          attributeKey,
-          error: errorMessage,
-          operation: 'processQueue'
-        });
-        queuedOperations.splice(i, 1);
-        continue;
-      }
-
-      // For relationship attributes, ensure the related collection exists
-      let canProcess = true;
-      if (operation.attribute.type === "relationship") {
-        const relatedCollection = operation.attribute.relatedCollection;
-        if (relatedCollection) {
-          // Try to resolve related collection by ID first, then by name
-          let relatedFound = false;
-
-          try {
-            await tryAwaitWithRetry(
-              async () => {
-                if ('getMetadata' in db && typeof db.getMetadata === 'function') {
-                  // DatabaseAdapter
-                  return (await (db as DatabaseAdapter).getTable({ databaseId: dbId, tableId: relatedCollection })).data;
-                } else {
-                  // Legacy Databases
-                  return await (db as Databases).getCollection(dbId, relatedCollection);
-                }
-              }
-            );
-            relatedFound = true;
-            nameToIdMapping.set(relatedCollection, relatedCollection); // Cache the ID mapping
-          } catch (_) {
-            // Try by name lookup
-            const cachedId = nameToIdMapping.get(relatedCollection);
-            if (cachedId) {
-              try {
-                await tryAwaitWithRetry(
-                  async () => {
-                    if ('getMetadata' in db && typeof db.getMetadata === 'function') {
-                      // DatabaseAdapter
-                      return (await (db as DatabaseAdapter).getTable({ databaseId: dbId, tableId: cachedId })).data;
-                    } else {
-                      // Legacy Databases
-                      return await (db as Databases).getCollection(dbId, cachedId);
-                    }
-                  }
-                );
-                relatedFound = true;
-              } catch (_) {
-                nameToIdMapping.delete(relatedCollection); // Remove stale cache
-              }
-            }
-
-            if (!relatedFound) {
-              // Final attempt: search by name
-              try {
-                const collections = 'getMetadata' in db && typeof db.getMetadata === 'function'
-                  ? await (db as DatabaseAdapter).listTables({ databaseId: dbId, queries: [Query.equal("name", relatedCollection)] })
-                  : await (db as Databases).listCollections(dbId, [Query.equal("name", relatedCollection)]);
-
-                if (collections.total && collections.total > 0) {
-                  const firstCollection = 'getMetadata' in db && typeof db.getMetadata === 'function'
-                    ? (collections as any).tables?.[0]
-                    : (collections as any).collections?.[0];
-                  nameToIdMapping.set(relatedCollection, firstCollection.$id);
-                  relatedFound = true;
-                }
-              } catch (_) {
-                // Related collection truly doesn't exist yet
-              }
-            }
-          }
-
-          if (!relatedFound) {
-            MessageFormatter.warning(
-              `Related collection '${relatedCollection}' not ready for attribute '${attributeKey}', keeping in queue`
-            );
-            canProcess = false;
-          }
-        }
-      }
-
-      if (canProcess && targetCollection) {
-        MessageFormatter.progress(
-          `Processing queued ${operation.attribute.type} attribute: '${attributeKey}' for collection: '${targetCollection.name}'`
-        );
-
-        const success = await createOrUpdateAttributeWithStatusCheck(
-          db,
-          dbId,
-          targetCollection,
-          operation.attribute
-        );
-
-        if (success) {
-          MessageFormatter.success(`Successfully processed queued attribute: '${attributeKey}'`);
-          logger.info('Queued attribute processed successfully', {
-            attributeKey,
-            collectionId,
-            targetCollectionName: targetCollection.name,
-            operation: 'processQueue'
-          });
-          markAttributeProcessed(dbId, collectionId, attributeKey);
-          queuedOperations.splice(i, 1);
-          progress = true;
-        } else {
-          MessageFormatter.error(`Failed to process queued attribute: '${attributeKey}', removing from queue`);
-          logger.error('Failed to process queued attribute', {
-            attributeKey,
-            collectionId,
-            targetCollectionName: targetCollection.name,
-            operation: 'processQueue'
-          });
-          queuedOperations.splice(i, 1);
-        }
-      }
-    }
-
-    if (queuedOperations.length === 0) {
-      break;
-    }
-
-    MessageFormatter.info(`Remaining operations after attempt ${attempts}: ${queuedOperations.length}`);
-  }
-
-  if (queuedOperations.length > 0) {
-    MessageFormatter.warning(
-      `${queuedOperations.length} operations remain unresolved after ${maxAttempts} attempts:`
-    );
-    queuedOperations.forEach((op, index) => {
-      MessageFormatter.warning(
-        `  ${index + 1}. ${op.attribute?.type} attribute '${op.attribute?.key}' for collection ${op.collectionId}`
-      );
-    });
-    MessageFormatter.warning("These may have unmet dependencies or require manual intervention");
-  } else {
-    MessageFormatter.success("All queued operations processed successfully");
-  }
-
-  MessageFormatter.section(`Surgical queue processing complete for ${dbId}`);
-};

package/src/shared/operationsTable.ts

@@ -1,338 +0,0 @@
-import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
-import { logger } from "./logging.js";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
-import { Query, ID } from "node-appwrite";
-import {
-	OPERATIONS_TABLE_ID,
-	OPERATIONS_TABLE_NAME,
-	type OperationRecord,
-	type OperationStatus,
-	OperationRecordSchema,
-} from "./operationsTableSchema.js";
-
-/**
- * Operations Table Manager
- *
- * Manages dynamic operations tracking tables in each database.
- * Creates _appwrite_operations table on-demand for tracking imports, exports, transfers, etc.
- */
-
-/**
- * Checks if operations table exists in database
- */
-async function tableExists(
-	db: DatabaseAdapter,
-	databaseId: string,
-): Promise<boolean> {
-	try {
-		await db.getTable({ databaseId, tableId: OPERATIONS_TABLE_ID });
-		return true;
-	} catch (error) {
-		return false;
-	}
-}
-
-/**
- * Creates the operations tracking table in the specified database
- * Table is created with underscore prefix to indicate it's a system table
- */
-export async function createOperationsTable(
-	db: DatabaseAdapter,
-	databaseId: string,
-): Promise<void> {
-	// Check if table already exists
-	const exists = await tableExists(db, databaseId);
-	if (exists) {
-		logger.debug("Operations table already exists", {
-			databaseId,
-			tableId: OPERATIONS_TABLE_ID,
-		});
-		return;
-	}
-
-	logger.info("Creating operations tracking table", {
-		databaseId,
-		tableId: OPERATIONS_TABLE_ID,
-	});
-
-	// Create table
-	await tryAwaitWithRetry(async () => {
-		await db.createTable({
-			databaseId,
-			id: OPERATIONS_TABLE_ID,
-			name: OPERATIONS_TABLE_NAME,
-		});
-	});
-
-	// Create attributes with retry logic
-	const attributes = [
-		{
-			key: "operationType",
-			type: "string",
-			size: 50,
-			required: true,
-		},
-		{
-			key: "targetCollection",
-			type: "string",
-			size: 50,
-			required: false,
-		},
-		{
-			key: "status",
-			type: "enum",
-			elements: ["pending", "in_progress", "completed", "failed", "cancelled"],
-			required: true,
-			default: "pending",
-		},
-		{
-			key: "progress",
-			type: "integer",
-			required: true,
-			default: 0,
-		},
-		{
-			key: "total",
-			type: "integer",
-			required: true,
-			default: 0,
-		},
-		{
-			key: "data",
-			type: "string",
-			size: 1048576, // 1MB for serialized data
-			required: false,
-		},
-		{
-			key: "error",
-			type: "string",
-			size: 10000,
-			required: false,
-		},
-	];
-
-	for (const attr of attributes) {
-		await tryAwaitWithRetry(async () => {
-			await db.createAttribute({
-				databaseId,
-				tableId: OPERATIONS_TABLE_ID,
-				...attr,
-			});
-		});
-	}
-
-	logger.info("Operations tracking table created successfully", {
-		databaseId,
-		tableId: OPERATIONS_TABLE_ID,
-		attributes: attributes.length,
-	});
-}
-
-/**
- * Finds an existing operation or creates a new one
- * Useful for resuming interrupted operations
- */
-export async function findOrCreateOperation(
-	db: DatabaseAdapter,
-	databaseId: string,
-	operationType: string,
-	params?: Partial<OperationRecord>,
-): Promise<OperationRecord> {
-	// Ensure operations table exists
-	await createOperationsTable(db, databaseId);
-
-	// Try to find existing pending operation
-	try {
-		const queries = [
-			Query.equal("operationType", operationType),
-			Query.equal("status", "pending"),
-		];
-
-		if (params?.targetCollection) {
-			queries.push(Query.equal("targetCollection", params.targetCollection));
-		}
-
-		const response = await db.listRows({
-			databaseId,
-			tableId: OPERATIONS_TABLE_ID,
-			queries,
-		});
-
-		if (response.rows && response.rows.length > 0) {
-			logger.debug("Found existing operation", {
-				operationId: response.rows[0].$id,
-				operationType,
-			});
-			return response.rows[0] as OperationRecord;
-		}
-	} catch (error) {
-		logger.debug("No existing operation found, creating new one", {
-			operationType,
-			error: error instanceof Error ? error.message : String(error),
-		});
-	}
-
-	// Create new operation
-	const newOperation = {
-		operationType,
-		targetCollection: params?.targetCollection,
-		status: "pending" as OperationStatus,
-		progress: params?.progress ?? 0,
-		total: params?.total ?? 0,
-		data: params?.data ? JSON.stringify(params.data) : undefined,
-		error: params?.error,
-	};
-
-	const result = await db.createRow({
-		databaseId,
-		tableId: OPERATIONS_TABLE_ID,
-		id: ID.unique(),
-		data: newOperation,
-	});
-
-	logger.info("Created new operation", {
-		operationId: result.data?.$id,
-		operationType,
-	});
-
-	return result.data as OperationRecord;
-}
-
-/**
- * Updates an existing operation record
- */
-export async function updateOperation(
-	db: DatabaseAdapter,
-	databaseId: string,
-	operationId: string,
-	updates: Partial<OperationRecord>,
-): Promise<OperationRecord> {
-	// Prepare update data (exclude system fields like $id, $createdAt, $updatedAt)
-	const updateData: Record<string, any> = {};
-
-	if (updates.operationType !== undefined)
-		updateData.operationType = updates.operationType;
-	if (updates.targetCollection !== undefined)
-		updateData.targetCollection = updates.targetCollection;
-	if (updates.status !== undefined) updateData.status = updates.status;
-	if (updates.progress !== undefined) updateData.progress = updates.progress;
-	if (updates.total !== undefined) updateData.total = updates.total;
-	if (updates.error !== undefined) updateData.error = updates.error;
-	if (updates.data !== undefined) {
-		updateData.data =
-			typeof updates.data === "string"
-				? updates.data
-				: JSON.stringify(updates.data);
-	}
-
-	try {
-		const result = await db.updateRow({
-			databaseId,
-			tableId: OPERATIONS_TABLE_ID,
-			id: operationId,
-			data: updateData,
-		});
-
-		logger.debug("Updated operation", {
-			operationId,
-			updates: Object.keys(updateData),
-		});
-
-		return result.data as OperationRecord;
-	} catch (error) {
-		logger.error("Failed to update operation", {
-			operationId,
-			error: error instanceof Error ? error.message : String(error),
-		});
-		throw error;
-	}
-}
-
-/**
- * Gets a single operation by ID
- */
-export async function getOperation(
-	db: DatabaseAdapter,
-	databaseId: string,
-	operationId: string,
-): Promise<OperationRecord | null> {
-	try {
-		const result = await db.getRow({
-			databaseId,
-			tableId: OPERATIONS_TABLE_ID,
-			id: operationId,
-		});
-
-		return result.data as OperationRecord;
-	} catch (error) {
-		logger.debug("Operation not found", {
-			operationId,
-			error: error instanceof Error ? error.message : String(error),
-		});
-		return null;
-	}
-}
-
-/**
- * Cleans up old completed operations
- * @param olderThan - Optional date, defaults to operations older than 7 days
- * @returns Number of operations deleted
- */
-export async function cleanupOperations(
-	db: DatabaseAdapter,
-	databaseId: string,
-	olderThan?: Date,
-): Promise<number> {
-	const cutoffDate =
-		olderThan || new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); // 7 days ago
-	const cutoffIso = cutoffDate.toISOString();
-
-	try {
-		// Query for completed operations older than cutoff
-		const response = await db.listRows({
-			databaseId,
-			tableId: OPERATIONS_TABLE_ID,
-			queries: [
-				Query.equal("status", ["completed", "failed", "cancelled"]),
-				Query.lessThan("$createdAt", cutoffIso),
-			],
-		});
-
-		if (!response.rows || response.rows.length === 0) {
-			logger.debug("No old operations to clean up", { databaseId });
-			return 0;
-		}
-
-		// Delete in batches
-		let deletedCount = 0;
-		for (const operation of response.rows) {
-			try {
-				await db.deleteRow({
-					databaseId,
-					tableId: OPERATIONS_TABLE_ID,
-					id: operation.$id,
-				});
-				deletedCount++;
-			} catch (error) {
-				logger.warn("Failed to delete operation", {
-					operationId: operation.$id,
-					error: error instanceof Error ? error.message : String(error),
-				});
-			}
-		}
-
-		logger.info("Cleaned up old operations", {
-			databaseId,
-			deletedCount,
-			cutoffDate: cutoffIso,
-		});
-
-		return deletedCount;
-	} catch (error) {
-		logger.error("Failed to cleanup operations", {
-			databaseId,
-			error: error instanceof Error ? error.message : String(error),
-		});
-		return 0;
-	}
-}