appwrite-utils-cli 1.9.7 → 1.12.0

package/src/migrations/importDataActions.ts

@@ -1,315 +0,0 @@
-import { type Databases, type Storage } from "node-appwrite";
-import type { AppwriteConfig } from "appwrite-utils";
-import {
-  validationRules,
-  type ValidationRules,
-  type AttributeMappings,
-} from "appwrite-utils";
-import { converterFunctions, type ConverterFunctions } from "appwrite-utils";
-import { convertObjectBySchema } from "../utils/dataConverters.js";
-import { type AfterImportActions } from "appwrite-utils";
-import { afterImportActions } from "./afterImportActions.js";
-import { logger } from "../shared/logging.js";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
-
-export class ImportDataActions {
-  private db: Databases;
-  private storage: Storage;
-  private config: AppwriteConfig;
-  private converterDefinitions: ConverterFunctions;
-  private validityRuleDefinitions: ValidationRules;
-  private afterImportActionsDefinitions: AfterImportActions;
-
-  constructor(
-    db: Databases,
-    storage: Storage,
-    config: AppwriteConfig,
-    converterDefinitions: ConverterFunctions,
-    validityRuleDefinitions: ValidationRules,
-    afterImportActionsDefinitions: AfterImportActions
-  ) {
-    this.db = db;
-    this.storage = storage;
-    this.config = config;
-    this.converterDefinitions = converterDefinitions;
-    this.validityRuleDefinitions = validityRuleDefinitions;
-    this.afterImportActionsDefinitions = afterImportActionsDefinitions;
-  }
-
-  /**
-   * Runs converter functions on the item based on the provided attribute mappings.
-   *
-   * @param item - The item to be transformed.
-   * @param attributeMappings - The mappings that define how each attribute should be transformed.
-   * @returns The transformed item.
-   */
-  runConverterFunctions(item: any, attributeMappings: AttributeMappings) {
-    const conversionSchema = attributeMappings.reduce((schema, mapping) => {
-      schema[mapping.targetKey] = (originalValue: any) => {
-        if (!mapping.converters) {
-          return originalValue;
-        }
-        return mapping.converters?.reduce((value, converterName) => {
-          let shouldProcessAsArray = false;
-          if (
-            (converterName.includes("[Arr]") ||
-              converterName.includes("[arr]")) &&
-            Array.isArray(value)
-          ) {
-            shouldProcessAsArray = true;
-            converterName = converterName
-              .replace("[Arr]", "")
-              .replace("[arr]", "");
-          } else if (
-            (!Array.isArray(value) && converterName.includes("[Arr]")) ||
-            converterName.includes("[arr]")
-          ) {
-            converterName = converterName
-              .replace("[Arr]", "")
-              .replace("[arr]", "");
-          }
-          const converterFunction =
-            converterFunctions[
-              converterName as keyof typeof converterFunctions
-            ];
-          if (converterFunction) {
-            if (Array.isArray(value) && !shouldProcessAsArray) {
-              return value.map((item) => converterFunction(item));
-            } else {
-              return converterFunction(value);
-            }
-          } else {
-            logger.warn(
-              `Converter function '${converterName}' is not defined.`
-            );
-            return value;
-          }
-        }, originalValue);
-      };
-      return schema;
-    }, {} as Record<string, (value: any) => any>);
-
-    // Convert the item using the constructed schema
-    const convertedItem = convertObjectBySchema(item, conversionSchema);
-    // Merge the converted item back into the original item object
-    Object.assign(item, convertedItem);
-    return item;
-  }
-
-  /**
-   * Validates a single data item based on defined validation rules.
-   * @param item The data item to validate.
-   * @param attributeMap The attribute mappings for the data item.
-   * @param context The context for resolving templated parameters in validation rules.
-   * @returns A promise that resolves to true if the item is valid, false otherwise.
-   */
-  validateItem(
-    item: any,
-    attributeMap: AttributeMappings,
-    context: { [key: string]: any }
-  ): boolean {
-    for (const mapping of attributeMap) {
-      const { validationActions } = mapping;
-      if (
-        !validationActions ||
-        !Array.isArray(validationActions) ||
-        !validationActions.length
-      ) {
-        return true; // Assume items without validation actions as valid.
-      }
-      for (const ruleDef of validationActions) {
-        const { action, params } = ruleDef;
-        const validationRule =
-          validationRules[action as keyof typeof validationRules];
-
-        if (!validationRule) {
-          logger.warn(`Validation rule '${action}' is not defined.`);
-          continue; // Optionally, consider undefined rules as a validation failure.
-        }
-
-        // Resolve templated parameters
-        const resolvedParams = params.map((param: any) =>
-          this.resolveTemplate(param, context, item)
-        );
-
-        // Apply the validation rule
-        let isValid = false;
-        if (Array.isArray(item)) {
-          isValid = item.every((item) =>
-            (validationRule as any)(item, ...resolvedParams)
-          );
-        } else {
-          isValid = (validationRule as any)(item, ...resolvedParams);
-        }
-        if (!isValid) {
-          logger.error(
-            `Validation failed for rule '${action}' with params ${params.join(
-              ", "
-            )}`
-          );
-          return false; // Stop validation on first failure
-        }
-      }
-    }
-
-    return true; // The item passed all validations
-  }
-
-  async executeAfterImportActions(
-    item: any,
-    attributeMap: AttributeMappings,
-    context: { [key: string]: any }
-  ): Promise<void> {
-    for (const mapping of attributeMap) {
-      const { postImportActions } = mapping;
-      if (!postImportActions || !Array.isArray(postImportActions)) {
-        continue; // Skip to the next attribute if no actions are defined
-      }
-      for (const actionDef of postImportActions) {
-        const { action, params } = actionDef;
-        console.log(
-          `Executing post-import action '${action}' for attribute '${
-            mapping.targetKey
-          }' with params ${params.join(", ")}...`
-        );
-        try {
-          await tryAwaitWithRetry(
-            async () => await this.executeAction(action, params, context, item)
-          );
-        } catch (error) {
-          logger.error(
-            `Failed to execute post-import action '${action}' for attribute '${mapping.targetKey}':`,
-            error
-          );
-        }
-      }
-    }
-  }
-
-  async executeAction(
-    actionName: string,
-    params: any[], // Accepts any type, including objects
-    context: { [key: string]: any },
-    item: any
-  ): Promise<void> {
-    const actionMethod =
-      afterImportActions[actionName as keyof typeof afterImportActions];
-    if (typeof actionMethod === "function") {
-      try {
-        // Resolve parameters, handling both strings and objects
-        const resolvedParams = params.map((param) => {
-          // Directly resolve each param, whether it's an object or a string
-          return this.resolveTemplate(param, context, item);
-        });
-
-        // Execute the action with resolved parameters
-        // Parameters are passed as-is, with objects treated as single parameters
-        console.log(
-          `Executing action '${actionName}' from context with params:`,
-          resolvedParams
-        );
-        logger.info(
-          `Executing action '${actionName}' from context: ${JSON.stringify(
-            context,
-            null,
-            2
-          )} with params:`,
-          resolvedParams
-        );
-        await (actionMethod as any)(this.config, ...resolvedParams);
-      } catch (error: any) {
-        logger.error(
-          `Error executing action '${actionName}' with context:`,
-          context,
-          error
-        );
-        throw new Error(
-          `Execution failed for action '${actionName}': ${error.message}`
-        );
-      }
-    } else {
-      logger.warn(`Action '${actionName}' is not defined.`);
-      throw new Error(`Action '${actionName}' is not defined.`);
-    }
-  }
-
-  /**
-   * Resolves a templated string or object using the provided context and current data item.
-   * If the template is a string that starts and ends with "{}", it replaces it with the corresponding value from item or context.
-   * If the template is an object, it recursively resolves its properties.
-   * @param template The templated string or object.
-   * @param context The context for resolving the template.
-   * @param item The current data item being processed.
-   */
-  resolveTemplate(
-    template: any,
-    context: { [key: string]: any },
-    item: any
-  ): any {
-    // Function to recursively resolve paths, including handling [any] notation
-    const resolvePath = (path: string, currentContext: any): any => {
-      const anyKeyRegex = /\[any\]/g;
-      let pathParts = path.split(".").filter(Boolean);
-
-      return pathParts.reduce((acc, part, index) => {
-        // Handle [any] part by iterating over all elements if it's an object or an array
-        if (part === "[any]") {
-          if (Array.isArray(acc)) {
-            return acc
-              .map((item) => item[pathParts[index + 1]])
-              .filter((item) => item !== undefined);
-          } else if (typeof acc === "object") {
-            return Object.values(acc)
-              .map((item: any) => item[pathParts[index + 1]])
-              .filter((item) => item !== undefined);
-          }
-        } else {
-          return acc?.[part];
-        }
-      }, currentContext);
-    };
-
-    if (typeof template === "string") {
-      // Matches placeholders in the template
-      const regex = /\{([^}]+)\}/g;
-      let match;
-      let resolvedString = template;
-      while ((match = regex.exec(template)) !== null) {
-        const path = match[1];
-        // Resolve the path, handling [any] notation and arrays/objects
-        const resolvedValue = resolvePath(path, { ...context, ...item });
-        if (resolvedValue !== undefined) {
-          // If it's an array (from [any] notation), join the values; adjust as needed
-          const value = Array.isArray(resolvedValue)
-            ? resolvedValue.join(", ")
-            : resolvedValue;
-          resolvedString = resolvedString.replace(match[0], value);
-        } else {
-          logger.warn(
-            `Failed to resolve ${template} in context: `,
-            JSON.stringify({ ...context, ...item }, null, 2)
-          );
-        }
-      }
-      // console.log(`Resolved string: ${resolvedString}`);
-      return resolvedString;
-    } else if (typeof template === "object" && template !== null) {
-      // Recursively resolve templates for each property in the object
-      const resolvedObject: any = Array.isArray(template) ? [] : {};
-      for (const key in template) {
-        const resolvedValue = this.resolveTemplate(
-          template[key],
-          context,
-          item
-        );
-        if (resolvedValue !== undefined) {
-          // Only assign if resolvedValue is not undefined
-          resolvedObject[key] = resolvedValue;
-        }
-      }
-      return resolvedObject;
-    }
-    // console.log(`Template is not a string or object: ${template}`);
-    return template;
-  }
-}

package/src/migrations/relationships.ts

@@ -1,334 +0,0 @@
-import { Databases, Query, type Models } from "node-appwrite";
-import { fetchAllCollections } from "../collections/methods.js";
-import type {
-  AppwriteConfig,
-  Attribute,
-  RelationshipAttribute,
-} from "appwrite-utils";
-import { logger } from "../shared/logging.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
-
-/**
- * Finds collections that have defined relationship attributes.
- */
-export const findCollectionsWithRelationships = (config: AppwriteConfig) => {
-  const toReturn = new Map<string, RelationshipAttribute[]>();
-  if (!config.collections) {
-    return toReturn;
-  }
-  for (const collection of config.collections) {
-    if (collection.attributes) {
-      for (const attribute of collection.attributes) {
-        if (
-          attribute.type === "relationship" &&
-          attribute.twoWay &&
-          attribute.side === "parent"
-        ) {
-          toReturn.set(collection.name, toReturn.get(collection.name) || []);
-          toReturn
-            .get(collection.name)
-            ?.push(attribute as RelationshipAttribute);
-        }
-      }
-    }
-  }
-  return toReturn;
-};
-
-export async function resolveAndUpdateRelationships(
-  dbId: string,
-  database: Databases,
-  config: AppwriteConfig
-) {
-  const collections = await fetchAllCollections(dbId, database);
-  const collectionsWithRelationships = findCollectionsWithRelationships(config);
-
-  // Process each collection sequentially
-  for (const collection of collections) {
-    MessageFormatter.processing(
-      `Processing collection: ${collection.name} (${collection.$id})`,
-      { prefix: "Migration" }
-    );
-    const relAttributeMap = collectionsWithRelationships.get(
-      collection.name
-    ) as RelationshipAttribute[]; // Get the relationship attributes for the collections
-
-    if (!relAttributeMap) {
-      MessageFormatter.info(
-        `No mapping found for collection: ${collection.name}, skipping...`,
-        { prefix: "Migration" }
-      );
-      continue;
-    }
-
-    await processCollection(dbId, database, collection, relAttributeMap);
-  }
-  MessageFormatter.success(
-    `Completed relationship resolution and update for database ID: ${dbId}`,
-    { prefix: "Migration" }
-  );
-}
-
-async function processCollection(
-  dbId: string,
-  database: Databases,
-  collection: Models.Collection,
-  relAttributeMap: RelationshipAttribute[]
-) {
-  let after; // For pagination
-  let hasMore = true;
-
-  while (hasMore) {
-    const response: Models.DocumentList<Models.Document> =
-      await database.listDocuments(dbId, collection.$id, [
-        Query.limit(100), // Fetch documents in batches of 100
-        ...(after ? [Query.cursorAfter(after)] : []),
-      ]);
-
-    const documents = response.documents;
-    MessageFormatter.info(
-      `Fetched ${documents.length} documents from collection: ${collection.name}`,
-      { prefix: "Migration" }
-    );
-
-    if (documents.length > 0) {
-      const updates = await prepareDocumentUpdates(
-        database,
-        dbId,
-        collection.name,
-        documents,
-        relAttributeMap
-      );
-
-      // Execute updates for the current batch
-      await executeUpdatesInBatches(dbId, database, updates);
-    }
-
-    if (documents.length === 100) {
-      after = documents[documents.length - 1].$id; // Prepare for the next page
-    } else {
-      hasMore = false; // No more documents to fetch
-    }
-  }
-}
-
-async function findDocumentsByOriginalId(
-  database: Databases,
-  dbId: string,
-  targetCollection: Models.Collection,
-  targetKey: string,
-  originalId: string | string[]
-): Promise<Models.Document[] | undefined> {
-  const relatedCollectionId = targetCollection.$id;
-  const collection = await database.listCollections(dbId, [
-    Query.equal("$id", relatedCollectionId),
-  ]);
-  if (collection.total === 0) {
-    MessageFormatter.warning(
-      `Collection ${relatedCollectionId} doesn't exist, skipping...`,
-      { prefix: "Migration" }
-    );
-    return undefined;
-  }
-  const targetAttr = collection.collections[0].attributes.find(
-    // @ts-ignore
-    (attr) => attr.key === targetKey
-  ) as any;
-  if (!targetAttr) {
-    MessageFormatter.warning(
-      `Attribute ${targetKey} not found in collection ${relatedCollectionId}, skipping...`,
-      { prefix: "Migration" }
-    );
-    return undefined;
-  }
-  let queries: string[] = [];
-  if (targetAttr.array) {
-    // @ts-ignore
-    queries.push(Query.contains(targetKey, originalId));
-  } else {
-    queries.push(Query.equal(targetKey, originalId));
-  }
-  const response = await database.listDocuments(dbId, relatedCollectionId, [
-    ...queries,
-    Query.limit(500), // Adjust the limit based on your needs or implement pagination
-  ]);
-
-  if (response.documents.length < 0) {
-    return undefined;
-  } else if (response.documents.length > 0) {
-    return response.documents;
-  } else {
-    return undefined;
-  }
-}
-
-async function prepareDocumentUpdates(
-  database: Databases,
-  dbId: string,
-  collectionName: string,
-  documents: Models.Document[],
-  relationships: RelationshipAttribute[]
-): Promise<{ collectionId: string; documentId: string; updatePayload: any }[]> {
-  MessageFormatter.processing(`Preparing updates for collection: ${collectionName}`, { prefix: "Migration" });
-  const updates: {
-    collectionId: string;
-    documentId: string;
-    updatePayload: any;
-  }[] = [];
-
-  const thisCollection = (
-    await database.listCollections(dbId, [Query.equal("name", collectionName)])
-  ).collections[0];
-  const thisCollectionId = thisCollection?.$id;
-
-  if (!thisCollectionId) {
-    MessageFormatter.warning(`No collection found with name: ${collectionName}`, { prefix: "Migration" });
-    return [];
-  }
-
-  for (const doc of documents) {
-    let updatePayload: { [key: string]: any } = {};
-
-    for (const rel of relationships) {
-      // Skip if not dealing with the parent side of a two-way relationship
-      if (rel.twoWay && rel.side !== "parent") {
-        MessageFormatter.info("Skipping non-parent side of two-way relationship...", { prefix: "Migration" });
-        continue;
-      }
-
-      const isSingleReference =
-        rel.relationType === "oneToOne" || rel.relationType === "manyToOne";
-      const originalIdField = rel.importMapping?.originalIdField;
-      const targetField = rel.importMapping?.targetField || originalIdField; // Use originalIdField if targetField is not specified
-      if (!originalIdField) {
-        MessageFormatter.warning("Missing originalIdField in importMapping, skipping...", { prefix: "Migration" });
-        continue;
-      }
-      const originalId = doc[originalIdField as keyof typeof doc];
-      if (!originalId) {
-        continue;
-      }
-
-      const relatedCollection = (
-        await database.listCollections(dbId, [
-          Query.equal("name", rel.relatedCollection),
-        ])
-      ).collections[0];
-
-      if (!relatedCollection) {
-        MessageFormatter.warning(
-          `Related collection ${rel.relatedCollection} not found, skipping...`,
-          { prefix: "Migration" }
-        );
-        continue;
-      }
-
-      const foundDocuments = await findDocumentsByOriginalId(
-        database,
-        dbId,
-        relatedCollection,
-        targetField!,
-        String(originalId)
-      );
-
-      if (foundDocuments && foundDocuments.length > 0) {
-        const relationshipKey = rel.key;
-        const existingRefs = doc[relationshipKey as keyof typeof doc] || [];
-        let existingRefIds: string[] = [];
-        if (Array.isArray(existingRefs)) {
-          // @ts-ignore
-          existingRefIds = existingRefs.map((ref) => ref.$id);
-        } else if (existingRefs) {
-          // @ts-ignore
-          existingRefIds = [existingRefs.$id];
-        }
-
-        const newRefs = foundDocuments.map((fd) => fd.$id);
-        const allRefs = [...new Set([...existingRefIds, ...newRefs])]; // Combine and remove duplicates
-
-        // Update logic based on the relationship cardinality
-        updatePayload[relationshipKey] = isSingleReference
-          ? newRefs[0] || existingRefIds[0]
-          : allRefs;
-        MessageFormatter.info(`Updating ${relationshipKey} with ${allRefs.length} refs`, { prefix: "Migration" });
-      }
-    }
-
-    if (Object.keys(updatePayload).length > 0) {
-      updates.push({
-        collectionId: thisCollectionId,
-        documentId: doc.$id,
-        updatePayload: updatePayload,
-      });
-    }
-  }
-
-  return updates;
-}
-
-async function processInBatches<T>(
-  items: T[],
-  batchSize: number,
-  processFunction: (batch: T[]) => Promise<void>
-) {
-  const maxParallelBatches = 25; // Adjust this value to control the number of parallel batches
-  let currentIndex = 0;
-  let activeBatchPromises: Promise<void>[] = [];
-
-  while (currentIndex < items.length) {
-    // While there's still data to process and we haven't reached our parallel limit
-    while (
-      currentIndex < items.length &&
-      activeBatchPromises.length < maxParallelBatches
-    ) {
-      const batch = items.slice(currentIndex, currentIndex + batchSize);
-      currentIndex += batchSize;
-      // Add new batch processing promise to the array
-      activeBatchPromises.push(processFunction(batch));
-    }
-
-    // Wait for one of the batch processes to complete
-    await Promise.race(activeBatchPromises).then(() => {
-      // Remove the resolved promise from the activeBatchPromises array
-      activeBatchPromises = activeBatchPromises.filter(
-        (p) => p !== Promise.race(activeBatchPromises)
-      );
-    });
-  }
-
-  // After processing all batches, ensure all active promises are resolved
-  await Promise.all(activeBatchPromises);
-}
-
-async function executeUpdatesInBatches(
-  dbId: string,
-  database: Databases,
-  updates: { collectionId: string; documentId: string; updatePayload: any }[]
-) {
-  const batchSize = 25; // Adjust based on your rate limit and performance testing
-  for (let i = 0; i < updates.length; i += batchSize) {
-    const batch = updates.slice(i, i + batchSize);
-    await Promise.all(
-      batch.map((update) =>
-        database
-          .updateDocument(
-            dbId,
-            update.collectionId,
-            update.documentId,
-            update.updatePayload
-          )
-          .catch((error) => {
-            logger.error(
-              `Error updating doc ${
-                update.documentId
-              } in ${dbId}, update payload: ${JSON.stringify(
-                update.updatePayload,
-                undefined,
-                4
-              )}, error: ${error}`
-            );
-          })
-      )
-    );
-  }
-}