appwrite-utils-cli 1.11.0 → 1.12.0

package/src/shared/migrationHelpers.ts

@@ -1,232 +0,0 @@
-import { ID, Query, type Databases } from "node-appwrite";
-import { BatchSchema, OperationSchema, type Operation } from "../storage/schemas.js";
-import { AttributeMappingsSchema } from "appwrite-utils";
-import { z } from "zod";
-import { logger } from 'appwrite-utils-helpers';
-import { tryAwaitWithRetry } from "appwrite-utils-helpers";
-import {
-  findOrCreateOperation as findOrCreateOp,
-  updateOperation as updateOp,
-  getOperation as getOp
-} from "./operationsTable.js";
-import type { DatabaseAdapter } from "appwrite-utils-helpers";
-import { MessageFormatter } from 'appwrite-utils-helpers';
-
-/**
- * Object that contains the context for an action that needs to be executed after import
- * Used in the afterImportActionsDefinitions
- * @type {ContextObject}
- * @typedef {Object} ContextObject
- * @property {string} collectionId - The ID of the collection
- * @property {any} finalItem - The final item that was imported
- * @property {string} action - The name of the action
- * @property {string[]} params - The parameters for the action
- * @property {Object} context - The context object for the action (all the data of this specific item)
- */
-export const ContextObject = z.object({
-  dbId: z.string(),
-  collectionId: z.string(),
-  finalItem: z.any(),
-  attributeMappings: AttributeMappingsSchema,
-  context: z.any(),
-});
-
-export type ContextObject = z.infer<typeof ContextObject>;
-
-export const createOrFindAfterImportOperation = async (
-  database: Databases,
-  collectionId: string,
-  context: ContextObject
-) => {
-  let operation = await findOrCreateOperationLegacy(
-    database,
-    collectionId,
-    "afterImportAction"
-  );
-  if (!operation.batches) {
-    operation.batches = [];
-  }
-
-  // Directly create a new batch for the context without checking for an existing batch
-  const contextData = JSON.stringify(context);
-  // Create a new batch with the contextData
-  const newBatchId = await addBatch(database, contextData);
-  // Update the operation with the new batch's $id
-  operation.batches = [...operation.batches, newBatchId];
-  await database.updateDocument(
-    "migrations",
-    "currentOperations",
-    operation.$id,
-    { batches: operation.batches }
-  );
-};
-
-export const addBatch = async (database: Databases, data: string) => {
-  const batch = await database.createDocument(
-    "migrations",
-    "batches",
-    ID.unique(),
-    {
-      data,
-      processed: false,
-    }
-  );
-  return batch.$id;
-};
-
-export const getAfterImportOperations = async (
-  database: Databases,
-  collectionId: string
-) => {
-  let lastDocumentId: string | undefined;
-  const allOperations = [];
-  let total = 0;
-
-  do {
-    const query = [
-      Query.equal("collectionId", collectionId),
-      Query.equal("operationType", "afterImportAction"),
-      Query.limit(100),
-    ];
-
-    if (lastDocumentId) {
-      query.push(Query.cursorAfter(lastDocumentId));
-    }
-
-    const operations = await database.listDocuments(
-      "migrations",
-      "currentOperations",
-      query
-    );
-    total = operations.total; // Update total with the latest fetch
-    allOperations.push(...operations.documents);
-
-    if (operations.documents.length > 0 && operations.documents.length >= 100) {
-      lastDocumentId =
-        operations.documents[operations.documents.length - 1].$id;
-    }
-  } while (allOperations.length < total);
-
-  const allOps = allOperations.map((op) => OperationSchema.parse(op));
-  return allOps;
-};
-
-// Legacy function for backward compatibility with old migrations database
-const findOrCreateOperationLegacy = async (
-  database: Databases,
-  collectionId: string,
-  operationType: string,
-  additionalQueries?: string[]
-) => {
-  const operations = await tryAwaitWithRetry(
-    async () =>
-      await database.listDocuments("migrations", "currentOperations", [
-        Query.equal("collectionId", collectionId),
-        Query.equal("operationType", operationType),
-        Query.equal("status", "pending"),
-        ...(additionalQueries || []),
-      ])
-  );
-
-  if (operations.documents.length > 0) {
-    return OperationSchema.parse(operations.documents[0]);
-  } else {
-    const op = await tryAwaitWithRetry(
-      async () =>
-        await database.createDocument(
-          "migrations",
-          "currentOperations",
-          ID.unique(),
-          {
-            operationType,
-            collectionId,
-            status: "pending",
-            batches: [],
-            progress: 0,
-            total: 0,
-            error: "",
-          }
-        )
-    );
-
-    return OperationSchema.parse(op);
-  }
-};
-
-export const findOrCreateOperation = async (
-  db: DatabaseAdapter,
-  databaseId: string,
-  operationType: string,
-  collectionId?: string,
-  data?: any
-): Promise<any> => {
-  // Use new operations table system
-  return await findOrCreateOp(db, databaseId, operationType, {
-    targetTable: collectionId,
-    data: data
-  });
-};
-
-export const updateOperation = async (
-  db: DatabaseAdapter,
-  databaseId: string,
-  operationId: string,
-  updates: any
-): Promise<any> => {
-  // Use new operations table system
-  return await updateOp(db, databaseId, operationId, updates);
-};
-
-export const getOperation = async (
-  db: DatabaseAdapter,
-  databaseId: string,
-  operationId: string
-): Promise<any> => {
-  // Use new operations table system
-  return await getOp(db, databaseId, operationId);
-};
-
-// Actual max 1073741824
-export const maxDataLength = 1073741820;
-export const maxBatchItems = 25;
-
-export const splitIntoBatches = (data: any[]): any[][] => {
-  let batches = [];
-  let currentBatch: any[] = [];
-  let currentBatchLength = 0;
-  let currentBatchItemCount = 0;
-
-  data.forEach((item, index) => {
-    const itemLength = JSON.stringify(item).length;
-    if (itemLength > maxDataLength) {
-      MessageFormatter.warning(
-        `Large item found at index ${index} with length ${itemLength}`,
-        { prefix: "Batch Splitter" }
-      );
-      logger.debug("Large item data:", item);
-    }
-    // Check if adding the current item would exceed the max length or max items per batch
-    if (
-      currentBatchLength + itemLength >= maxDataLength ||
-      currentBatchItemCount >= maxBatchItems
-    ) {
-      // If so, start a new batch
-      batches.push(currentBatch);
-      currentBatch = [item];
-      currentBatchLength = itemLength;
-      currentBatchItemCount = 1; // Reset item count for the new batch
-    } else {
-      // Otherwise, add the item to the current batch
-      currentBatch.push(item);
-      currentBatchLength += itemLength;
-      currentBatchItemCount++;
-    }
-  });
-
-  // Don't forget to add the last batch if it's not empty
-  if (currentBatch.length > 0) {
-    batches.push(currentBatch);
-  }
-
-  return batches;
-};

package/src/shared/operationQueue.ts

@@ -1,376 +0,0 @@
-import { Query, type Databases, type Models } from "node-appwrite";
-import type { Attribute } from "appwrite-utils";
-import { createOrUpdateAttributeWithStatusCheck } from "../collections/columns.js";
-import { fetchAndCacheCollectionByName } from "../collections/methods.js";
-import { tryAwaitWithRetry } from "appwrite-utils-helpers";
-import type { DatabaseAdapter } from "appwrite-utils-helpers";
-import { logger, MessageFormatter } from "appwrite-utils-helpers";
-
-export interface QueuedOperation {
-  type: "attribute";
-  collectionId?: string;
-  attribute?: Attribute;
-  collection?: Models.Collection;
-  dependencies?: string[];
-}
-
-// Global state management
-export const queuedOperations: QueuedOperation[] = [];
-export const nameToIdMapping: Map<string, string> = new Map();
-// Keys are scoped per database to avoid cross-database collisions
-// Collections key format: `${databaseId}::${collectionId}`
-// Attributes key format: `${databaseId}::${collectionId}::${attributeKey}`
-export const processedCollections: Set<string> = new Set();
-export const processedAttributes: Set<string> = new Set();
-
-// Helpers to build scoped keys
-const collectionKey = (databaseId: string, collectionId: string) => `${databaseId}::${collectionId}`;
-const attributeKeyScoped = (databaseId: string, collectionId: string, key: string) => `${databaseId}::${collectionId}::${key}`;
-
-export const enqueueOperation = (operation: QueuedOperation) => {
-  // Avoid duplicate queue entries for same attribute
-  const attributeKey = operation.attribute?.key;
-  const collectionId = operation.collectionId;
-
-  logger.info('Enqueueing operation', {
-    type: operation.type,
-    attributeKey,
-    collectionId,
-    dependencies: operation.dependencies,
-    queueSizeBefore: queuedOperations.length,
-    operation: 'enqueueOperation'
-  });
-
-  if (attributeKey && collectionId) {
-    const duplicateIndex = queuedOperations.findIndex(
-      (op) => op.collectionId === collectionId && op.attribute?.key === attributeKey
-    );
-
-    if (duplicateIndex !== -1) {
-      MessageFormatter.info(`Replacing existing queue entry for attribute: ${attributeKey}`);
-      logger.info('Replacing duplicate queue entry', {
-        attributeKey,
-        collectionId,
-        duplicateIndex,
-        operation: 'enqueueOperation'
-      });
-      queuedOperations[duplicateIndex] = operation;
-      return;
-    }
-  }
-
-  queuedOperations.push(operation);
-  logger.debug('Operation enqueued successfully', {
-    attributeKey,
-    collectionId,
-    queueSizeAfter: queuedOperations.length,
-    operation: 'enqueueOperation'
-  });
-};
-
-/**
- * Clear all caches and processing state - use between operations
- */
-export const clearProcessingState = () => {
-  const sizeBefore = {
-    collections: processedCollections.size,
-    attributes: processedAttributes.size,
-    nameMapping: nameToIdMapping.size
-  };
-
-  processedCollections.clear();
-  processedAttributes.clear();
-  nameToIdMapping.clear();
-
-  logger.debug("Cleared processing state caches", { operation: "clearProcessingState", sizeBefore });
-  logger.info('Processing state cleared', {
-    sizeBefore,
-    operation: 'clearProcessingState'
-  });
-};
-
-/**
- * Check if a collection has already been fully processed
- */
-export const isCollectionProcessed = (collectionId: string, databaseId: string): boolean => {
-  return processedCollections.has(collectionKey(databaseId, collectionId));
-};
-
-/**
- * Mark a collection as fully processed
- */
-export const markCollectionProcessed = (collectionId: string, collectionName: string | undefined, databaseId: string) => {
-  processedCollections.add(collectionKey(databaseId, collectionId));
-
-  const logData = {
-    databaseId,
-    collectionId,
-    collectionName,
-    totalProcessedCollections: processedCollections.size,
-    operation: 'markCollectionProcessed'
-  };
-
-  if (collectionName) {
-    MessageFormatter.success(`Marked collection '${collectionName}' (${collectionId}) as processed`, { prefix: 'Tables' });
-  }
-
-  logger.info('Collection marked as processed', logData);
-};
-
-/**
- * Check if a specific attribute has been processed
- */
-export const isAttributeProcessed = (databaseId: string, collectionId: string, attributeKey: string): boolean => {
-  return processedAttributes.has(attributeKeyScoped(databaseId, collectionId, attributeKey));
-};
-
-/**
- * Mark a specific attribute as processed
- */
-export const markAttributeProcessed = (databaseId: string, collectionId: string, attributeKey: string) => {
-  const identifier = attributeKeyScoped(databaseId, collectionId, attributeKey);
-  processedAttributes.add(identifier);
-
-  logger.debug('Attribute marked as processed', {
-    databaseId,
-    collectionId,
-    attributeKey,
-    identifier,
-    totalProcessedAttributes: processedAttributes.size,
-    operation: 'markAttributeProcessed'
-  });
-};
-
-/**
- * Process only specific attributes in the queue, not entire collections
- * This prevents triggering full collection re-processing cycles
- */
-export const processQueue = async (db: Databases | DatabaseAdapter, dbId: string) => {
-  const startTime = Date.now();
-
-  if (queuedOperations.length === 0) {
-    MessageFormatter.info("No queued operations to process");
-    logger.info('Queue processing skipped - no operations', {
-      dbId,
-      operation: 'processQueue'
-    });
-    return;
-  }
-
-  MessageFormatter.section(`Starting surgical queue processing of ${queuedOperations.length} operations for ${dbId}`);
-
-  logger.info('Starting queue processing', {
-    dbId,
-    queueSize: queuedOperations.length,
-    operations: queuedOperations.map(op => ({
-      type: op.type,
-      attributeKey: op.attribute?.key,
-      collectionId: op.collectionId,
-      dependencies: op.dependencies
-    })),
-    operation: 'processQueue'
-  });
-
-  let progress = true;
-  let attempts = 0;
-  const maxAttempts = 3; // Prevent infinite loops
-
-  while (progress && attempts < maxAttempts) {
-    progress = false;
-    attempts++;
-
-    MessageFormatter.info(`Queue processing attempt ${attempts}/${maxAttempts}`);
-
-    logger.info('Queue processing attempt started', {
-      attempt: attempts,
-      maxAttempts,
-      remainingOperations: queuedOperations.length,
-      dbId,
-      operation: 'processQueue'
-    });
-
-    for (let i = queuedOperations.length - 1; i >= 0; i--) {
-      const operation = queuedOperations[i];
-
-      if (!operation.attribute || !operation.collectionId) {
-        MessageFormatter.warning("Invalid operation, removing from queue");
-        queuedOperations.splice(i, 1);
-        continue;
-      }
-
-      const attributeKey = operation.attribute.key;
-      const collectionId = operation.collectionId;
-
-      // Skip if this specific attribute was already processed (per database)
-      if (isAttributeProcessed(dbId, collectionId, attributeKey)) {
-        MessageFormatter.debug(`Attribute '${attributeKey}' already processed, removing from queue`);
-        logger.debug('Removing already processed attribute from queue', {
-          attributeKey,
-          collectionId,
-          queueIndex: i,
-          operation: 'processQueue'
-        });
-        queuedOperations.splice(i, 1);
-        continue;
-      }
-
-      let targetCollection: Models.Collection | undefined;
-
-      // Resolve the target collection (where the attribute will be created)
-      try {
-        targetCollection = await tryAwaitWithRetry(
-          async () => {
-            if ('getMetadata' in db && typeof db.getMetadata === 'function') {
-              // DatabaseAdapter
-              return (await (db as DatabaseAdapter).getTable({ databaseId: dbId, tableId: collectionId })).data;
-            } else {
-              // Legacy Databases
-              return await (db as Databases).getCollection(dbId, collectionId);
-            }
-          }
-        );
-      } catch (e) {
-        const errorMessage = e instanceof Error ? e.message : String(e);
-        MessageFormatter.error(`Target collection ${collectionId} not found, removing from queue`);
-        logger.error('Target collection not found during queue processing', {
-          collectionId,
-          attributeKey,
-          error: errorMessage,
-          operation: 'processQueue'
-        });
-        queuedOperations.splice(i, 1);
-        continue;
-      }
-
-      // For relationship attributes, ensure the related collection exists
-      let canProcess = true;
-      if (operation.attribute.type === "relationship") {
-        const relatedCollection = operation.attribute.relatedCollection;
-        if (relatedCollection) {
-          // Try to resolve related collection by ID first, then by name
-          let relatedFound = false;
-
-          try {
-            await tryAwaitWithRetry(
-              async () => {
-                if ('getMetadata' in db && typeof db.getMetadata === 'function') {
-                  // DatabaseAdapter
-                  return (await (db as DatabaseAdapter).getTable({ databaseId: dbId, tableId: relatedCollection })).data;
-                } else {
-                  // Legacy Databases
-                  return await (db as Databases).getCollection(dbId, relatedCollection);
-                }
-              }
-            );
-            relatedFound = true;
-            nameToIdMapping.set(relatedCollection, relatedCollection); // Cache the ID mapping
-          } catch (_) {
-            // Try by name lookup
-            const cachedId = nameToIdMapping.get(relatedCollection);
-            if (cachedId) {
-              try {
-                await tryAwaitWithRetry(
-                  async () => {
-                    if ('getMetadata' in db && typeof db.getMetadata === 'function') {
-                      // DatabaseAdapter
-                      return (await (db as DatabaseAdapter).getTable({ databaseId: dbId, tableId: cachedId })).data;
-                    } else {
-                      // Legacy Databases
-                      return await (db as Databases).getCollection(dbId, cachedId);
-                    }
-                  }
-                );
-                relatedFound = true;
-              } catch (_) {
-                nameToIdMapping.delete(relatedCollection); // Remove stale cache
-              }
-            }
-
-            if (!relatedFound) {
-              // Final attempt: search by name
-              try {
-                const collections = 'getMetadata' in db && typeof db.getMetadata === 'function'
-                  ? await (db as DatabaseAdapter).listTables({ databaseId: dbId, queries: [Query.equal("name", relatedCollection)] })
-                  : await (db as Databases).listCollections(dbId, [Query.equal("name", relatedCollection)]);
-
-                if (collections.total && collections.total > 0) {
-                  const firstCollection = 'getMetadata' in db && typeof db.getMetadata === 'function'
-                    ? (collections as any).tables?.[0]
-                    : (collections as any).collections?.[0];
-                  nameToIdMapping.set(relatedCollection, firstCollection.$id);
-                  relatedFound = true;
-                }
-              } catch (_) {
-                // Related collection truly doesn't exist yet
-              }
-            }
-          }
-
-          if (!relatedFound) {
-            MessageFormatter.warning(
-              `Related collection '${relatedCollection}' not ready for attribute '${attributeKey}', keeping in queue`
-            );
-            canProcess = false;
-          }
-        }
-      }
-
-      if (canProcess && targetCollection) {
-        MessageFormatter.progress(
-          `Processing queued ${operation.attribute.type} attribute: '${attributeKey}' for collection: '${targetCollection.name}'`
-        );
-
-        const success = await createOrUpdateAttributeWithStatusCheck(
-          db,
-          dbId,
-          targetCollection,
-          operation.attribute
-        );
-
-        if (success) {
-          MessageFormatter.success(`Successfully processed queued attribute: '${attributeKey}'`);
-          logger.info('Queued attribute processed successfully', {
-            attributeKey,
-            collectionId,
-            targetCollectionName: targetCollection.name,
-            operation: 'processQueue'
-          });
-          markAttributeProcessed(dbId, collectionId, attributeKey);
-          queuedOperations.splice(i, 1);
-          progress = true;
-        } else {
-          MessageFormatter.error(`Failed to process queued attribute: '${attributeKey}', removing from queue`);
-          logger.error('Failed to process queued attribute', {
-            attributeKey,
-            collectionId,
-            targetCollectionName: targetCollection.name,
-            operation: 'processQueue'
-          });
-          queuedOperations.splice(i, 1);
-        }
-      }
-    }
-
-    if (queuedOperations.length === 0) {
-      break;
-    }
-
-    MessageFormatter.info(`Remaining operations after attempt ${attempts}: ${queuedOperations.length}`);
-  }
-
-  if (queuedOperations.length > 0) {
-    MessageFormatter.warning(
-      `${queuedOperations.length} operations remain unresolved after ${maxAttempts} attempts:`
-    );
-    queuedOperations.forEach((op, index) => {
-      MessageFormatter.warning(
-        `  ${index + 1}. ${op.attribute?.type} attribute '${op.attribute?.key}' for collection ${op.collectionId}`
-      );
-    });
-    MessageFormatter.warning("These may have unmet dependencies or require manual intervention");
-  } else {
-    MessageFormatter.success("All queued operations processed successfully");
-  }
-
-  MessageFormatter.section(`Surgical queue processing complete for ${dbId}`);
-};