@njdamstra/appwrite-utils-cli 1.8.9

package/src/migrations/importDataActions.ts

@@ -0,0 +1,315 @@
+import { type Databases, type Storage } from "node-appwrite";
+import type { AppwriteConfig } from "@njdamstra/appwrite-utils";
+import {
+  validationRules,
+  type ValidationRules,
+  type AttributeMappings,
+} from "@njdamstra/appwrite-utils";
+import { converterFunctions, type ConverterFunctions } from "@njdamstra/appwrite-utils";
+import { convertObjectBySchema } from "../utils/dataConverters.js";
+import { type AfterImportActions } from "@njdamstra/appwrite-utils";
+import { afterImportActions } from "./afterImportActions.js";
+import { logger } from "../shared/logging.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+
+export class ImportDataActions {
+  private db: Databases;
+  private storage: Storage;
+  private config: AppwriteConfig;
+  private converterDefinitions: ConverterFunctions;
+  private validityRuleDefinitions: ValidationRules;
+  private afterImportActionsDefinitions: AfterImportActions;
+
+  constructor(
+    db: Databases,
+    storage: Storage,
+    config: AppwriteConfig,
+    converterDefinitions: ConverterFunctions,
+    validityRuleDefinitions: ValidationRules,
+    afterImportActionsDefinitions: AfterImportActions
+  ) {
+    this.db = db;
+    this.storage = storage;
+    this.config = config;
+    this.converterDefinitions = converterDefinitions;
+    this.validityRuleDefinitions = validityRuleDefinitions;
+    this.afterImportActionsDefinitions = afterImportActionsDefinitions;
+  }
+
+  /**
+   * Runs converter functions on the item based on the provided attribute mappings.
+   *
+   * @param item - The item to be transformed.
+   * @param attributeMappings - The mappings that define how each attribute should be transformed.
+   * @returns The transformed item.
+   */
+  runConverterFunctions(item: any, attributeMappings: AttributeMappings) {
+    const conversionSchema = attributeMappings.reduce((schema, mapping) => {
+      schema[mapping.targetKey] = (originalValue: any) => {
+        if (!mapping.converters) {
+          return originalValue;
+        }
+        return mapping.converters?.reduce((value, converterName) => {
+          let shouldProcessAsArray = false;
+          if (
+            (converterName.includes("[Arr]") ||
+              converterName.includes("[arr]")) &&
+            Array.isArray(value)
+          ) {
+            shouldProcessAsArray = true;
+            converterName = converterName
+              .replace("[Arr]", "")
+              .replace("[arr]", "");
+          } else if (
+            (!Array.isArray(value) && converterName.includes("[Arr]")) ||
+            converterName.includes("[arr]")
+          ) {
+            converterName = converterName
+              .replace("[Arr]", "")
+              .replace("[arr]", "");
+          }
+          const converterFunction =
+            converterFunctions[
+              converterName as keyof typeof converterFunctions
+            ];
+          if (converterFunction) {
+            if (Array.isArray(value) && !shouldProcessAsArray) {
+              return value.map((item) => converterFunction(item));
+            } else {
+              return converterFunction(value);
+            }
+          } else {
+            logger.warn(
+              `Converter function '${converterName}' is not defined.`
+            );
+            return value;
+          }
+        }, originalValue);
+      };
+      return schema;
+    }, {} as Record<string, (value: any) => any>);
+
+    // Convert the item using the constructed schema
+    const convertedItem = convertObjectBySchema(item, conversionSchema);
+    // Merge the converted item back into the original item object
+    Object.assign(item, convertedItem);
+    return item;
+  }
+
+  /**
+   * Validates a single data item based on defined validation rules.
+   * @param item The data item to validate.
+   * @param attributeMap The attribute mappings for the data item.
+   * @param context The context for resolving templated parameters in validation rules.
+   * @returns A promise that resolves to true if the item is valid, false otherwise.
+   */
+  validateItem(
+    item: any,
+    attributeMap: AttributeMappings,
+    context: { [key: string]: any }
+  ): boolean {
+    for (const mapping of attributeMap) {
+      const { validationActions } = mapping;
+      if (
+        !validationActions ||
+        !Array.isArray(validationActions) ||
+        !validationActions.length
+      ) {
+        return true; // Assume items without validation actions as valid.
+      }
+      for (const ruleDef of validationActions) {
+        const { action, params } = ruleDef;
+        const validationRule =
+          validationRules[action as keyof typeof validationRules];
+
+        if (!validationRule) {
+          logger.warn(`Validation rule '${action}' is not defined.`);
+          continue; // Optionally, consider undefined rules as a validation failure.
+        }
+
+        // Resolve templated parameters
+        const resolvedParams = params.map((param: any) =>
+          this.resolveTemplate(param, context, item)
+        );
+
+        // Apply the validation rule
+        let isValid = false;
+        if (Array.isArray(item)) {
+          isValid = item.every((item) =>
+            (validationRule as any)(item, ...resolvedParams)
+          );
+        } else {
+          isValid = (validationRule as any)(item, ...resolvedParams);
+        }
+        if (!isValid) {
+          logger.error(
+            `Validation failed for rule '${action}' with params ${params.join(
+              ", "
+            )}`
+          );
+          return false; // Stop validation on first failure
+        }
+      }
+    }
+
+    return true; // The item passed all validations
+  }
+
+  async executeAfterImportActions(
+    item: any,
+    attributeMap: AttributeMappings,
+    context: { [key: string]: any }
+  ): Promise<void> {
+    for (const mapping of attributeMap) {
+      const { postImportActions } = mapping;
+      if (!postImportActions || !Array.isArray(postImportActions)) {
+        continue; // Skip to the next attribute if no actions are defined
+      }
+      for (const actionDef of postImportActions) {
+        const { action, params } = actionDef;
+        console.log(
+          `Executing post-import action '${action}' for attribute '${
+            mapping.targetKey
+          }' with params ${params.join(", ")}...`
+        );
+        try {
+          await tryAwaitWithRetry(
+            async () => await this.executeAction(action, params, context, item)
+          );
+        } catch (error) {
+          logger.error(
+            `Failed to execute post-import action '${action}' for attribute '${mapping.targetKey}':`,
+            error
+          );
+        }
+      }
+    }
+  }
+
+  async executeAction(
+    actionName: string,
+    params: any[], // Accepts any type, including objects
+    context: { [key: string]: any },
+    item: any
+  ): Promise<void> {
+    const actionMethod =
+      afterImportActions[actionName as keyof typeof afterImportActions];
+    if (typeof actionMethod === "function") {
+      try {
+        // Resolve parameters, handling both strings and objects
+        const resolvedParams = params.map((param) => {
+          // Directly resolve each param, whether it's an object or a string
+          return this.resolveTemplate(param, context, item);
+        });
+
+        // Execute the action with resolved parameters
+        // Parameters are passed as-is, with objects treated as single parameters
+        console.log(
+          `Executing action '${actionName}' from context with params:`,
+          resolvedParams
+        );
+        logger.info(
+          `Executing action '${actionName}' from context: ${JSON.stringify(
+            context,
+            null,
+            2
+          )} with params:`,
+          resolvedParams
+        );
+        await (actionMethod as any)(this.config, ...resolvedParams);
+      } catch (error: any) {
+        logger.error(
+          `Error executing action '${actionName}' with context:`,
+          context,
+          error
+        );
+        throw new Error(
+          `Execution failed for action '${actionName}': ${error.message}`
+        );
+      }
+    } else {
+      logger.warn(`Action '${actionName}' is not defined.`);
+      throw new Error(`Action '${actionName}' is not defined.`);
+    }
+  }
+
+  /**
+   * Resolves a templated string or object using the provided context and current data item.
+   * If the template is a string that starts and ends with "{}", it replaces it with the corresponding value from item or context.
+   * If the template is an object, it recursively resolves its properties.
+   * @param template The templated string or object.
+   * @param context The context for resolving the template.
+   * @param item The current data item being processed.
+   */
+  resolveTemplate(
+    template: any,
+    context: { [key: string]: any },
+    item: any
+  ): any {
+    // Function to recursively resolve paths, including handling [any] notation
+    const resolvePath = (path: string, currentContext: any): any => {
+      const anyKeyRegex = /\[any\]/g;
+      let pathParts = path.split(".").filter(Boolean);
+
+      return pathParts.reduce((acc, part, index) => {
+        // Handle [any] part by iterating over all elements if it's an object or an array
+        if (part === "[any]") {
+          if (Array.isArray(acc)) {
+            return acc
+              .map((item) => item[pathParts[index + 1]])
+              .filter((item) => item !== undefined);
+          } else if (typeof acc === "object") {
+            return Object.values(acc)
+              .map((item: any) => item[pathParts[index + 1]])
+              .filter((item) => item !== undefined);
+          }
+        } else {
+          return acc?.[part];
+        }
+      }, currentContext);
+    };
+
+    if (typeof template === "string") {
+      // Matches placeholders in the template
+      const regex = /\{([^}]+)\}/g;
+      let match;
+      let resolvedString = template;
+      while ((match = regex.exec(template)) !== null) {
+        const path = match[1];
+        // Resolve the path, handling [any] notation and arrays/objects
+        const resolvedValue = resolvePath(path, { ...context, ...item });
+        if (resolvedValue !== undefined) {
+          // If it's an array (from [any] notation), join the values; adjust as needed
+          const value = Array.isArray(resolvedValue)
+            ? resolvedValue.join(", ")
+            : resolvedValue;
+          resolvedString = resolvedString.replace(match[0], value);
+        } else {
+          logger.warn(
+            `Failed to resolve ${template} in context: `,
+            JSON.stringify({ ...context, ...item }, null, 2)
+          );
+        }
+      }
+      // console.log(`Resolved string: ${resolvedString}`);
+      return resolvedString;
+    } else if (typeof template === "object" && template !== null) {
+      // Recursively resolve templates for each property in the object
+      const resolvedObject: any = Array.isArray(template) ? [] : {};
+      for (const key in template) {
+        const resolvedValue = this.resolveTemplate(
+          template[key],
+          context,
+          item
+        );
+        if (resolvedValue !== undefined) {
+          // Only assign if resolvedValue is not undefined
+          resolvedObject[key] = resolvedValue;
+        }
+      }
+      return resolvedObject;
+    }
+    // console.log(`Template is not a string or object: ${template}`);
+    return template;
+  }
+}

package/src/migrations/relationships.ts

@@ -0,0 +1,334 @@
+import { Databases, Query, type Models } from "node-appwrite";
+import { fetchAllCollections } from "../collections/methods.js";
+import type {
+  AppwriteConfig,
+  Attribute,
+  RelationshipAttribute,
+} from "@njdamstra/appwrite-utils";
+import { logger } from "../shared/logging.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+
+/**
+ * Finds collections that have defined relationship attributes.
+ */
+export const findCollectionsWithRelationships = (config: AppwriteConfig) => {
+  const toReturn = new Map<string, RelationshipAttribute[]>();
+  if (!config.collections) {
+    return toReturn;
+  }
+  for (const collection of config.collections) {
+    if (collection.attributes) {
+      for (const attribute of collection.attributes) {
+        if (
+          attribute.type === "relationship" &&
+          attribute.twoWay &&
+          attribute.side === "parent"
+        ) {
+          toReturn.set(collection.name, toReturn.get(collection.name) || []);
+          toReturn
+            .get(collection.name)
+            ?.push(attribute as RelationshipAttribute);
+        }
+      }
+    }
+  }
+  return toReturn;
+};
+
+export async function resolveAndUpdateRelationships(
+  dbId: string,
+  database: Databases,
+  config: AppwriteConfig
+) {
+  const collections = await fetchAllCollections(dbId, database);
+  const collectionsWithRelationships = findCollectionsWithRelationships(config);
+
+  // Process each collection sequentially
+  for (const collection of collections) {
+    MessageFormatter.processing(
+      `Processing collection: ${collection.name} (${collection.$id})`,
+      { prefix: "Migration" }
+    );
+    const relAttributeMap = collectionsWithRelationships.get(
+      collection.name
+    ) as RelationshipAttribute[]; // Get the relationship attributes for the collections
+
+    if (!relAttributeMap) {
+      MessageFormatter.info(
+        `No mapping found for collection: ${collection.name}, skipping...`,
+        { prefix: "Migration" }
+      );
+      continue;
+    }
+
+    await processCollection(dbId, database, collection, relAttributeMap);
+  }
+  MessageFormatter.success(
+    `Completed relationship resolution and update for database ID: ${dbId}`,
+    { prefix: "Migration" }
+  );
+}
+
+async function processCollection(
+  dbId: string,
+  database: Databases,
+  collection: Models.Collection,
+  relAttributeMap: RelationshipAttribute[]
+) {
+  let after; // For pagination
+  let hasMore = true;
+
+  while (hasMore) {
+    const response: Models.DocumentList<Models.Document> =
+      await database.listDocuments(dbId, collection.$id, [
+        Query.limit(100), // Fetch documents in batches of 100
+        ...(after ? [Query.cursorAfter(after)] : []),
+      ]);
+
+    const documents = response.documents;
+    MessageFormatter.info(
+      `Fetched ${documents.length} documents from collection: ${collection.name}`,
+      { prefix: "Migration" }
+    );
+
+    if (documents.length > 0) {
+      const updates = await prepareDocumentUpdates(
+        database,
+        dbId,
+        collection.name,
+        documents,
+        relAttributeMap
+      );
+
+      // Execute updates for the current batch
+      await executeUpdatesInBatches(dbId, database, updates);
+    }
+
+    if (documents.length === 100) {
+      after = documents[documents.length - 1].$id; // Prepare for the next page
+    } else {
+      hasMore = false; // No more documents to fetch
+    }
+  }
+}
+
+async function findDocumentsByOriginalId(
+  database: Databases,
+  dbId: string,
+  targetCollection: Models.Collection,
+  targetKey: string,
+  originalId: string | string[]
+): Promise<Models.Document[] | undefined> {
+  const relatedCollectionId = targetCollection.$id;
+  const collection = await database.listCollections(dbId, [
+    Query.equal("$id", relatedCollectionId),
+  ]);
+  if (collection.total === 0) {
+    MessageFormatter.warning(
+      `Collection ${relatedCollectionId} doesn't exist, skipping...`,
+      { prefix: "Migration" }
+    );
+    return undefined;
+  }
+  const targetAttr = collection.collections[0].attributes.find(
+    // @ts-ignore
+    (attr) => attr.key === targetKey
+  ) as any;
+  if (!targetAttr) {
+    MessageFormatter.warning(
+      `Attribute ${targetKey} not found in collection ${relatedCollectionId}, skipping...`,
+      { prefix: "Migration" }
+    );
+    return undefined;
+  }
+  let queries: string[] = [];
+  if (targetAttr.array) {
+    // @ts-ignore
+    queries.push(Query.contains(targetKey, originalId));
+  } else {
+    queries.push(Query.equal(targetKey, originalId));
+  }
+  const response = await database.listDocuments(dbId, relatedCollectionId, [
+    ...queries,
+    Query.limit(500), // Adjust the limit based on your needs or implement pagination
+  ]);
+
+  if (response.documents.length < 0) {
+    return undefined;
+  } else if (response.documents.length > 0) {
+    return response.documents;
+  } else {
+    return undefined;
+  }
+}
+
+async function prepareDocumentUpdates(
+  database: Databases,
+  dbId: string,
+  collectionName: string,
+  documents: Models.Document[],
+  relationships: RelationshipAttribute[]
+): Promise<{ collectionId: string; documentId: string; updatePayload: any }[]> {
+  MessageFormatter.processing(`Preparing updates for collection: ${collectionName}`, { prefix: "Migration" });
+  const updates: {
+    collectionId: string;
+    documentId: string;
+    updatePayload: any;
+  }[] = [];
+
+  const thisCollection = (
+    await database.listCollections(dbId, [Query.equal("name", collectionName)])
+  ).collections[0];
+  const thisCollectionId = thisCollection?.$id;
+
+  if (!thisCollectionId) {
+    MessageFormatter.warning(`No collection found with name: ${collectionName}`, { prefix: "Migration" });
+    return [];
+  }
+
+  for (const doc of documents) {
+    let updatePayload: { [key: string]: any } = {};
+
+    for (const rel of relationships) {
+      // Skip if not dealing with the parent side of a two-way relationship
+      if (rel.twoWay && rel.side !== "parent") {
+        MessageFormatter.info("Skipping non-parent side of two-way relationship...", { prefix: "Migration" });
+        continue;
+      }
+
+      const isSingleReference =
+        rel.relationType === "oneToOne" || rel.relationType === "manyToOne";
+      const originalIdField = rel.importMapping?.originalIdField;
+      const targetField = rel.importMapping?.targetField || originalIdField; // Use originalIdField if targetField is not specified
+      if (!originalIdField) {
+        MessageFormatter.warning("Missing originalIdField in importMapping, skipping...", { prefix: "Migration" });
+        continue;
+      }
+      const originalId = doc[originalIdField as keyof typeof doc];
+      if (!originalId) {
+        continue;
+      }
+
+      const relatedCollection = (
+        await database.listCollections(dbId, [
+          Query.equal("name", rel.relatedCollection),
+        ])
+      ).collections[0];
+
+      if (!relatedCollection) {
+        MessageFormatter.warning(
+          `Related collection ${rel.relatedCollection} not found, skipping...`,
+          { prefix: "Migration" }
+        );
+        continue;
+      }
+
+      const foundDocuments = await findDocumentsByOriginalId(
+        database,
+        dbId,
+        relatedCollection,
+        targetField!,
+        String(originalId)
+      );
+
+      if (foundDocuments && foundDocuments.length > 0) {
+        const relationshipKey = rel.key;
+        const existingRefs = doc[relationshipKey as keyof typeof doc] || [];
+        let existingRefIds: string[] = [];
+        if (Array.isArray(existingRefs)) {
+          // @ts-ignore
+          existingRefIds = existingRefs.map((ref) => ref.$id);
+        } else if (existingRefs) {
+          // @ts-ignore
+          existingRefIds = [existingRefs.$id];
+        }
+
+        const newRefs = foundDocuments.map((fd) => fd.$id);
+        const allRefs = [...new Set([...existingRefIds, ...newRefs])]; // Combine and remove duplicates
+
+        // Update logic based on the relationship cardinality
+        updatePayload[relationshipKey] = isSingleReference
+          ? newRefs[0] || existingRefIds[0]
+          : allRefs;
+        MessageFormatter.info(`Updating ${relationshipKey} with ${allRefs.length} refs`, { prefix: "Migration" });
+      }
+    }
+
+    if (Object.keys(updatePayload).length > 0) {
+      updates.push({
+        collectionId: thisCollectionId,
+        documentId: doc.$id,
+        updatePayload: updatePayload,
+      });
+    }
+  }
+
+  return updates;
+}
+
+async function processInBatches<T>(
+  items: T[],
+  batchSize: number,
+  processFunction: (batch: T[]) => Promise<void>
+) {
+  const maxParallelBatches = 25; // Adjust this value to control the number of parallel batches
+  let currentIndex = 0;
+  let activeBatchPromises: Promise<void>[] = [];
+
+  while (currentIndex < items.length) {
+    // While there's still data to process and we haven't reached our parallel limit
+    while (
+      currentIndex < items.length &&
+      activeBatchPromises.length < maxParallelBatches
+    ) {
+      const batch = items.slice(currentIndex, currentIndex + batchSize);
+      currentIndex += batchSize;
+      // Add new batch processing promise to the array
+      activeBatchPromises.push(processFunction(batch));
+    }
+
+    // Wait for one of the batch processes to complete
+    await Promise.race(activeBatchPromises).then(() => {
+      // Remove the resolved promise from the activeBatchPromises array
+      activeBatchPromises = activeBatchPromises.filter(
+        (p) => p !== Promise.race(activeBatchPromises)
+      );
+    });
+  }
+
+  // After processing all batches, ensure all active promises are resolved
+  await Promise.all(activeBatchPromises);
+}
+
+async function executeUpdatesInBatches(
+  dbId: string,
+  database: Databases,
+  updates: { collectionId: string; documentId: string; updatePayload: any }[]
+) {
+  const batchSize = 25; // Adjust based on your rate limit and performance testing
+  for (let i = 0; i < updates.length; i += batchSize) {
+    const batch = updates.slice(i, i + batchSize);
+    await Promise.all(
+      batch.map((update) =>
+        database
+          .updateDocument(
+            dbId,
+            update.collectionId,
+            update.documentId,
+            update.updatePayload
+          )
+          .catch((error) => {
+            logger.error(
+              `Error updating doc ${
+                update.documentId
+              } in ${dbId}, update payload: ${JSON.stringify(
+                update.updatePayload,
+                undefined,
+                4
+              )}, error: ${error}`
+            );
+          })
+      )
+    );
+  }
+}