appwrite-utils-cli 0.0.1

package/src/migrations/importDataActions.ts

@@ -0,0 +1,313 @@
+import {
+  ID,
+  InputFile,
+  Query,
+  type Databases,
+  type Storage,
+} from "node-appwrite";
+import type { AppwriteConfig } from "./schema.js";
+import { validationRules, type ValidationRules } from "./validationRules.js";
+import {
+  converterFunctions,
+  convertObjectBySchema,
+  type ConverterFunctions,
+} from "./converters.js";
+import {
+  afterImportActions,
+  type AfterImportActions,
+} from "./afterImportActions.js";
+import { logger } from "./logging.js";
+
+type AttributeMappings =
+  AppwriteConfig["collections"][number]["importDefs"][number]["attributeMappings"];
+
+export class ImportDataActions {
+  private db: Databases;
+  private storage: Storage;
+  private config: AppwriteConfig;
+  private converterDefinitions: ConverterFunctions;
+  private validityRuleDefinitions: ValidationRules;
+  private afterImportActionsDefinitions: AfterImportActions;
+
+  constructor(
+    db: Databases,
+    storage: Storage,
+    config: AppwriteConfig,
+    converterDefinitions: ConverterFunctions,
+    validityRuleDefinitions: ValidationRules,
+    afterImportActionsDefinitions: AfterImportActions
+  ) {
+    this.db = db;
+    this.storage = storage;
+    this.config = config;
+    this.converterDefinitions = converterDefinitions;
+    this.validityRuleDefinitions = validityRuleDefinitions;
+    this.afterImportActionsDefinitions = afterImportActionsDefinitions;
+  }
+
+  async runConverterFunctions(item: any, attributeMappings: AttributeMappings) {
+    const conversionSchema = attributeMappings.reduce((schema, mapping) => {
+      schema[mapping.targetKey] = (originalValue: any) => {
+        return mapping.converters.reduce((value, converterName) => {
+          let shouldProcessAsArray = false;
+          if (
+            (converterName.includes("[Arr]") ||
+              converterName.includes("[arr]")) &&
+            Array.isArray(value)
+          ) {
+            shouldProcessAsArray = true;
+            converterName = converterName
+              .replace("[Arr]", "")
+              .replace("[arr]", "");
+          } else if (
+            (!Array.isArray(value) && converterName.includes("[Arr]")) ||
+            converterName.includes("[arr]")
+          ) {
+            converterName = converterName
+              .replace("[Arr]", "")
+              .replace("[arr]", "");
+          }
+          const converterFunction =
+            converterFunctions[
+              converterName as keyof typeof converterFunctions
+            ];
+          if (converterFunction) {
+            if (Array.isArray(value) && !shouldProcessAsArray) {
+              return value.map((item) => converterFunction(item));
+            } else {
+              return converterFunction(value);
+            }
+          } else {
+            logger.warn(
+              `Converter function '${converterName}' is not defined.`
+            );
+            return value;
+          }
+        }, originalValue);
+      };
+      return schema;
+    }, {} as Record<string, (value: any) => any>);
+
+    // Convert the item using the constructed schema
+    const convertedItem = convertObjectBySchema(item, conversionSchema);
+    // Merge the converted item back into the original item object
+    Object.assign(item, convertedItem);
+    console.log("Converted item:", item);
+    return item;
+  }
+
+  /**
+   * Validates a single data item based on defined validation rules.
+   * @param item The data item to validate.
+   * @param context The context for resolving templated parameters in validation rules.
+   * @returns A promise that resolves to true if the item is valid, false otherwise.
+   */
+  async validateItem(
+    item: any,
+    attributeMap: AttributeMappings,
+    context: { [key: string]: any }
+  ): Promise<boolean> {
+    for (const mapping of attributeMap) {
+      const { validationActions } = mapping;
+      if (
+        !validationActions ||
+        !Array.isArray(validationActions) ||
+        !validationActions.length
+      ) {
+        console.warn(
+          "No validation actions defined for the item, assuming true"
+        );
+        return true; // Assume items without validation actions as valid.
+      }
+      for (const ruleDef of validationActions) {
+        const { action, params } = ruleDef;
+        const validationRule =
+          validationRules[action as keyof typeof validationRules];
+
+        if (!validationRule) {
+          logger.warn(`Validation rule '${action}' is not defined.`);
+          continue; // Optionally, consider undefined rules as a validation failure.
+        }
+
+        // Resolve templated parameters
+        const resolvedParams = params.map((param: any) =>
+          this.resolveTemplate(param, context, item)
+        );
+
+        // Apply the validation rule
+        let isValid = false;
+        if (Array.isArray(item)) {
+          isValid = item.every((item) =>
+            (validationRule as any)(item, ...resolvedParams)
+          );
+        } else {
+          isValid = (validationRule as any)(item, ...resolvedParams);
+        }
+        if (!isValid) {
+          logger.error(
+            `Validation failed for rule '${action}' with params ${params.join(
+              ", "
+            )}`
+          );
+          return false; // Stop validation on first failure
+        }
+      }
+    }
+
+    return true; // The item passed all validations
+  }
+
+  async executeAfterImportActions(
+    item: any,
+    attributeMap: AttributeMappings,
+    context: { [key: string]: any }
+  ): Promise<void> {
+    for (const mapping of attributeMap) {
+      const { postImportActions } = mapping;
+      if (!postImportActions || !Array.isArray(postImportActions)) {
+        console.warn(
+          `No post-import actions defined for attribute: ${mapping.targetKey}`,
+          postImportActions
+        );
+        continue; // Skip to the next attribute if no actions are defined
+      }
+      for (const actionDef of postImportActions) {
+        const { action, params } = actionDef;
+        console.log(
+          `Executing post-import action '${action}' for attribute '${
+            mapping.targetKey
+          }' with params ${params.join(", ")}...`
+        );
+        logger.info(
+          `Executing post-import action '${action}' for attribute '${
+            mapping.targetKey
+          }' with params ${params.join(", ")}...`
+        );
+        try {
+          await this.executeAction(action, params, context, item);
+        } catch (error) {
+          logger.error(
+            `Failed to execute post-import action '${action}' for attribute '${mapping.targetKey}':`,
+            error
+          );
+        }
+      }
+    }
+  }
+
+  async executeAction(
+    actionName: string,
+    params: any[], // Accepts any type, including objects
+    context: { [key: string]: any },
+    item: any
+  ): Promise<void> {
+    const actionMethod =
+      afterImportActions[actionName as keyof typeof afterImportActions];
+    if (typeof actionMethod === "function") {
+      try {
+        // Resolve parameters, handling both strings and objects
+        const resolvedParams = params.map((param) => {
+          // Directly resolve each param, whether it's an object or a string
+          return this.resolveTemplate(param, context, item);
+        });
+
+        // Execute the action with resolved parameters
+        // Parameters are passed as-is, with objects treated as single parameters
+        logger.info(
+          `Executing action '${actionName}' from context: ${JSON.stringify(
+            context,
+            null,
+            2
+          )} with params:`,
+          resolvedParams
+        );
+        await (actionMethod as any)(this.config, ...resolvedParams);
+      } catch (error: any) {
+        logger.error(`Error executing action '${actionName}':`, error);
+        throw new Error(
+          `Execution failed for action '${actionName}': ${error.message}`
+        );
+      }
+    } else {
+      logger.warn(`Action '${actionName}' is not defined.`);
+      throw new Error(`Action '${actionName}' is not defined.`);
+    }
+  }
+
+  /**
+   * Resolves a templated string or object using the provided context and current data item.
+   * If the template is a string that starts and ends with "{}", it replaces it with the corresponding value from item or context.
+   * If the template is an object, it recursively resolves its properties.
+   * @param template The templated string or object.
+   * @param context The context for resolving the template.
+   * @param item The current data item being processed.
+   */
+  resolveTemplate(
+    template: any,
+    context: { [key: string]: any },
+    item: any
+  ): any {
+    // Function to recursively resolve paths, including handling [any] notation
+    const resolvePath = (path: string, currentContext: any): any => {
+      const anyKeyRegex = /\[any\]/g;
+      let pathParts = path.split(".").filter(Boolean);
+
+      return pathParts.reduce((acc, part, index) => {
+        // Handle [any] part by iterating over all elements if it's an object or an array
+        if (part === "[any]") {
+          if (Array.isArray(acc)) {
+            return acc
+              .map((item) => item[pathParts[index + 1]])
+              .filter((item) => item !== undefined);
+          } else if (typeof acc === "object") {
+            return Object.values(acc)
+              .map((item: any) => item[pathParts[index + 1]])
+              .filter((item) => item !== undefined);
+          }
+        } else {
+          return acc?.[part];
+        }
+      }, currentContext);
+    };
+
+    if (typeof template === "string") {
+      // Matches placeholders in the template
+      const regex = /\{([^}]+)\}/g;
+      let match;
+      let resolvedString = template;
+      while ((match = regex.exec(template)) !== null) {
+        const path = match[1];
+        // Resolve the path, handling [any] notation and arrays/objects
+        const resolvedValue = resolvePath(path, { ...context, ...item });
+        if (resolvedValue !== undefined) {
+          // If it's an array (from [any] notation), join the values; adjust as needed
+          const value = Array.isArray(resolvedValue)
+            ? resolvedValue.join(", ")
+            : resolvedValue;
+          resolvedString = resolvedString.replace(match[0], value);
+        } else {
+          logger.warn(`Failed to resolve ${template} in context: `, context);
+        }
+      }
+      // console.log(`Resolved string: ${resolvedString}`);
+      return resolvedString;
+    } else if (typeof template === "object" && template !== null) {
+      // Recursively resolve templates for each property in the object
+      const resolvedObject: any = Array.isArray(template) ? [] : {};
+      for (const key in template) {
+        const resolvedValue = this.resolveTemplate(
+          template[key],
+          context,
+          item
+        );
+        if (resolvedValue !== undefined) {
+          // Only assign if resolvedValue is not undefined
+          resolvedObject[key] = resolvedValue;
+        }
+      }
+      return resolvedObject;
+    }
+    // console.log(`Template is not a string or object: ${template}`);
+    return template;
+  }
+}

package/src/migrations/indexes.ts

@@ -0,0 +1,37 @@
+import { indexSchema, type Index } from "./schema.js";
+import { Databases, Query, type Models } from "node-appwrite";
+// import {}
+
+export const createOrUpdateIndex = async (
+  dbId: string,
+  db: Databases,
+  collectionId: string,
+  index: Index
+) => {
+  const existingIndex = await db.listIndexes(dbId, collectionId, [
+    Query.equal("key", index.key),
+  ]);
+  if (existingIndex.total > 0) {
+    await db.deleteIndex(dbId, collectionId, existingIndex.indexes[0].key);
+  }
+  const newIndex = await db.createIndex(
+    dbId,
+    collectionId,
+    index.key,
+    index.type,
+    index.attributes,
+    index.orders
+  );
+  return newIndex;
+};
+
+export const createOrUpdateIndexes = async (
+  dbId: string,
+  db: Databases,
+  collectionId: string,
+  indexes: Index[]
+) => {
+  for (const index of indexes) {
+    await createOrUpdateIndex(dbId, db, collectionId, index);
+  }
+};

package/src/migrations/logging.ts

@@ -0,0 +1,15 @@
+import winston from "winston";
+
+export const logger = winston.createLogger({
+  level: "info",
+  format: winston.format.prettyPrint(),
+  defaultMeta: { service: "appwrite-utils-cli" },
+  transports: [
+    //
+    // - Write all logs with importance level of `error` or less to `error.log`
+    // - Write all logs with importance level of `info` or less to `combined.log`
+    //
+    new winston.transports.File({ filename: "error.log", level: "error" }),
+    new winston.transports.File({ filename: "combined.log" }),
+  ],
+});

package/src/migrations/migrationHelper.ts

@@ -0,0 +1,100 @@
+import { ID, Query, type Databases } from "node-appwrite";
+import { OperationSchema } from "./backup.js";
+
+export const findOrCreateOperation = async (
+  database: Databases,
+  collectionId: string,
+  operationType: string
+) => {
+  // Here you would query your database for an existing operation
+  // If it doesn't exist, create a new one
+  // This is a simplified example
+  const operations = await database.listDocuments(
+    "migrations",
+    "currentOperations",
+    [
+      Query.equal("collectionId", collectionId),
+      Query.equal("operationType", operationType),
+      Query.equal("status", "in_progress"),
+    ]
+  );
+
+  if (operations.documents.length > 0) {
+    return OperationSchema.parse(operations.documents[0]); // Assuming the first document is the operation we want
+  } else {
+    // Create a new operation document
+    const op = await database.createDocument(
+      "migrations",
+      "currentOperations",
+      ID.unique(),
+      {
+        operationType,
+        collectionId,
+        status: "in_progress",
+        batches: [],
+        progress: 0,
+        total: 0,
+        error: "",
+      }
+    );
+
+    return OperationSchema.parse(op);
+  }
+};
+
+export const updateOperation = async (
+  database: Databases,
+  operationId: string,
+  updateFields: any
+) => {
+  await database.updateDocument(
+    "migrations",
+    "currentOperations",
+    operationId,
+    updateFields
+  );
+};
+
+// Actual max 1073741824
+export const maxDataLength = 1073741820;
+export const maxBatchItems = 100;
+
+export const splitIntoBatches = (data: any[]): any[][] => {
+  let batches = [];
+  let currentBatch: any[] = [];
+  let currentBatchLength = 0;
+  let currentBatchItemCount = 0;
+
+  data.forEach((item, index) => {
+    const itemLength = JSON.stringify(item).length;
+    if (itemLength > maxDataLength) {
+      console.log(
+        item,
+        `Large item found at index ${index} with length ${itemLength}:`
+      );
+    }
+    // Check if adding the current item would exceed the max length or max items per batch
+    if (
+      currentBatchLength + itemLength >= maxDataLength ||
+      currentBatchItemCount >= maxBatchItems
+    ) {
+      // If so, start a new batch
+      batches.push(currentBatch);
+      currentBatch = [item];
+      currentBatchLength = itemLength;
+      currentBatchItemCount = 1; // Reset item count for the new batch
+    } else {
+      // Otherwise, add the item to the current batch
+      currentBatch.push(item);
+      currentBatchLength += itemLength;
+      currentBatchItemCount++;
+    }
+  });
+
+  // Don't forget to add the last batch if it's not empty
+  if (currentBatch.length > 0) {
+    batches.push(currentBatch);
+  }
+
+  return batches;
+};

package/src/migrations/queue.ts

@@ -0,0 +1,119 @@
+import { Query, type Databases, type Models } from "node-appwrite";
+import type { Attribute } from "./schema.js";
+import { createOrUpdateAttribute } from "./attributes.js";
+import _ from "lodash";
+import { fetchAndCacheCollectionByName } from "./collections.js";
+
+export interface QueuedOperation {
+  type: "attribute";
+  collectionId?: string;
+  attribute?: Attribute;
+  collection?: Models.Collection;
+  dependencies?: string[];
+}
+export const queuedOperations: QueuedOperation[] = [];
+export const nameToIdMapping: Map<string, string> = new Map();
+
+export const enqueueOperation = (operation: QueuedOperation) => {
+  queuedOperations.push(operation);
+};
+
+export const processQueue = async (db: Databases, dbId: string) => {
+  console.log("---------------------------------");
+  console.log(`Starting Queue processing of ${dbId}`);
+  console.log("---------------------------------");
+  let progress = true;
+
+  while (progress) {
+    progress = false;
+    console.log("Processing queued operations:");
+    for (let i = 0; i < queuedOperations.length; i++) {
+      const operation = queuedOperations[i];
+      let collectionFound: Models.Collection | undefined;
+
+      // Handle relationship attribute operations
+      if (operation.attribute?.type === "relationship") {
+        // Attempt to resolve the collection directly if collectionId is specified
+        if (operation.collectionId) {
+          console.log(`\tFetching collection by ID: ${operation.collectionId}`);
+          try {
+            collectionFound = await db.getCollection(
+              dbId,
+              operation.collectionId
+            );
+          } catch (e) {
+            console.log(
+              `\tCollection not found by ID: ${operation.collectionId}`
+            );
+          }
+        }
+        // Attempt to resolve related collection if specified and not already found
+        if (!collectionFound && operation.attribute?.relatedCollection) {
+          collectionFound = await fetchAndCacheCollectionByName(
+            db,
+            dbId,
+            operation.attribute.relatedCollection
+          );
+        }
+        // Handle dependencies if collection still not found
+        if (!collectionFound) {
+          for (const dep of operation.dependencies || []) {
+            collectionFound = await fetchAndCacheCollectionByName(
+              db,
+              dbId,
+              dep
+            );
+            if (collectionFound) break; // Break early if collection is found
+          }
+        }
+      } else if (operation.collectionId) {
+        // Handle non-relationship operations with a specified collectionId
+        console.log(`\tFetching collection by ID: ${operation.collectionId}`);
+        try {
+          collectionFound = await db.getCollection(
+            dbId,
+            operation.collectionId
+          );
+        } catch (e) {
+          console.log(
+            `\tCollection not found by ID: ${operation.collectionId}`
+          );
+        }
+      }
+
+      // Process the operation if the collection is found
+      if (collectionFound && operation.attribute) {
+        console.log(
+          `\tProcessing attribute: ${operation.attribute.key} for collection ID: ${collectionFound.$id}`
+        );
+        await createOrUpdateAttribute(
+          db,
+          dbId,
+          collectionFound,
+          operation.attribute
+        );
+        queuedOperations.splice(i, 1);
+        i--; // Adjust index since we're modifying the array
+        progress = true;
+      } else {
+        console.error(
+          `\tCollection not found for operation, removing from queue: ${JSON.stringify(
+            operation
+          )}`
+        );
+        queuedOperations.splice(i, 1);
+        i--; // Adjust index since we're modifying the array
+      }
+    }
+    console.log(`\tFinished processing queued operations`);
+  }
+
+  if (queuedOperations.length > 0) {
+    console.error("Unresolved operations remain due to unmet dependencies.");
+    console.log(queuedOperations);
+  }
+
+  console.log("---------------------------------");
+  console.log(`Queue processing complete for ${dbId}`);
+  console.log("---------------------------------");
+};