appwrite-utils-cli 0.0.2 → 0.0.4

package/src/appwrite/importData/dogs.json

@@ -0,0 +1,76 @@
+{
+  "RECORDS": [
+    {
+      "id": "1",
+      "name": "Charlie",
+      "breed": "Golden Retriever",
+      "age": 2,
+      "ownerId": "123",
+      "vetRecords": {
+        "1": {
+          "id": 1,
+          "visitDate": "2022-01-01",
+          "vetName": "Dr. Smith"
+        },
+        "2": {
+          "id": 2,
+          "visitDate": "2022-02-01",
+          "vetName": "Dr. Johnson"
+        },
+        "3": {
+          "id": 3,
+          "visitDate": "2022-03-01",
+          "vetName": "Dr. Williams"
+        }
+      }
+    },
+    {
+      "id": "2",
+      "name": "Buddy",
+      "breed": "Labrador",
+      "age": 3,
+      "ownerId": "123",
+      "vetRecords": {
+        "4": {
+          "id": 4,
+          "visitDate": "2022-01-02",
+          "vetName": "Dr. Smith"
+        },
+        "5": {
+          "id": 5,
+          "visitDate": "2022-02-02",
+          "vetName": "Dr. Johnson"
+        },
+        "6": {
+          "id": 6,
+          "visitDate": "2022-03-02",
+          "vetName": "Dr. Williams"
+        }
+      }
+    },
+    {
+      "id": "3",
+      "name": "Max",
+      "breed": "Poodle",
+      "age": 1,
+      "ownerId": "456",
+      "vetRecords": {
+        "7": {
+          "id": 7,
+          "visitDate": "2022-01-03",
+          "vetName": "Dr. Smith"
+        },
+        "8": {
+          "id": 8,
+          "visitDate": "2022-02-03",
+          "vetName": "Dr. Johnson"
+        },
+        "9": {
+          "id": 9,
+          "visitDate": "2022-03-03",
+          "vetName": "Dr. Williams"
+        }
+      }
+    }
+  ]
+}

package/src/appwrite/importData/members.json

@@ -0,0 +1,16 @@
+{
+  "RECORDS": [
+    {
+      "id": 123,
+      "name": "John Doe",
+      "email": "john@doe.com",
+      "photoUrl": "https://picsum.photos/200/300"
+    },
+    {
+      "id": 456,
+      "name": "Jane Doe",
+      "email": "jane@doe.com",
+      "photoUrl": "https://picsum.photos/200/300"
+    }
+  ]
+}

package/src/appwrite/schemas/dogs.ts

@@ -0,0 +1,27 @@
+import { z } from "zod";
+import { MembersSchema, type Members } from "./members";
+
+
+export const DogsSchemaBase = z.object({
+  $id: z.string().optional(),
+  $createdAt: z.date().or(z.string()).optional(),
+  $updatedAt: z.date().or(z.string()).optional(),
+  name: z.string().max(255, "Maximum length of 255 characters exceeded"),
+  breed: z.string().max(255, "Maximum length of 255 characters exceeded").nullish(),
+  age: z.number().int().min(0, "Minimum value of 0 not met").max(100, "Maximum value of 100 exceeded").nullish(),
+  idOrig: z.string().max(20, "Maximum length of 20 characters exceeded").nullish(),
+  ownerIdOrig: z.string().max(255, "Maximum length of 255 characters exceeded").nullish(),
+  vetRecords: z.string().max(255, "Maximum length of 255 characters exceeded").nullish(),
+  vetRecordIds: z.array(z.string().max(255, "Maximum length of 255 characters exceeded")).nullish(),
+});
+
+export type DogsBase = z.infer<typeof DogsSchemaBase> & {
+  owner?: Members | null;
+};
+
+export const DogsSchema: z.ZodType<DogsBase> = DogsSchemaBase.extend({
+  owner: z.lazy(() => MembersSchema.nullish()),
+});
+
+export type Dogs = z.infer<typeof DogsSchema>;
+

package/src/appwrite/schemas/members.ts

@@ -0,0 +1,24 @@
+import { z } from "zod";
+import { DogsSchema, type Dogs } from "./dogs";
+
+
+export const MembersSchemaBase = z.object({
+  $id: z.string().optional(),
+  $createdAt: z.date().or(z.string()).optional(),
+  $updatedAt: z.date().or(z.string()).optional(),
+  idOrig: z.string().max(255, "Maximum length of 255 characters exceeded").nullish(),
+  dogIds: z.array(z.string().max(255, "Maximum length of 255 characters exceeded")).nullish(),
+  profilePhoto: z.string().max(255, "Maximum length of 255 characters exceeded").nullish(),
+  profilePhotoTest: z.string().max(255, "Maximum length of 255 characters exceeded").nullish(),
+});
+
+export type MembersBase = z.infer<typeof MembersSchemaBase> & {
+  dogs?: Dogs[];
+};
+
+export const MembersSchema: z.ZodType<MembersBase> = MembersSchemaBase.extend({
+  dogs: z.lazy(() => DogsSchema.array().default([])),
+});
+
+export type Members = z.infer<typeof MembersSchema>;
+

package/src/migrations/afterImportActions.ts

@@ -258,6 +258,9 @@ export const afterImportActions = {
         } else {
           updateData = file.$id; // Set the new file ID
         }
+        await db.updateDocument(dbId, collId, doc.$id, {
+          [fieldName]: updateData,
+        });
         // console.log(
         //   "Updating document with file: ",
         //   doc.$id,

package/src/migrations/backup.ts

@@ -64,7 +64,16 @@ export const OperationSchema = z.object({
   progress: z.number(),
   total: z.number(),
   error: z.string(),
-  status: z.enum(["pending", "in_progress", "completed", "error"]),
+  status: z
+    .enum([
+      "pending",
+      "ready",
+      "in_progress",
+      "completed",
+      "error",
+      "cancelled",
+    ])
+    .default("pending"),
 });
 
 export type Operation = z.infer<typeof OperationSchema>;
@@ -133,7 +142,14 @@ export const getMigrationCollectionSchemas = () => {
     attributeSchema.parse({
       key: "status",
       type: "enum",
-      elements: ["pending", "in_progress", "completed", "error"],
+      elements: [
+        "pending",
+        "ready",
+        "in_progress",
+        "completed",
+        "error",
+        "cancelled",
+      ],
       error: "Invalid Status",
       array: false,
       xdefault: "pending",

package/src/migrations/importController.ts

@@ -24,6 +24,19 @@ import type { SetupOptions } from "../utilsController.js";
 import { resolveAndUpdateRelationships } from "./relationships.js";
 import { AuthUserCreateSchema } from "../types.js";
 import { UsersController } from "./users.js";
+import { logger } from "./logging.js";
+import {
+  ContextObject,
+  createOrFindAfterImportOperation,
+  getAfterImportOperations,
+  setAllPendingAfterImportActionsToReady,
+  updateOperation,
+} from "./migrationHelper.js";
+import {
+  BatchSchema,
+  OperationCreateSchema,
+  OperationSchema,
+} from "./backup.js";
 
 export class ImportController {
   private config: AppwriteConfig;
@@ -34,11 +47,11 @@ export class ImportController {
   private setupOptions: SetupOptions;
   private documentCache: Map<string, any>;
   private batchLimit: number = 25; // Define batch size limit
-  private postImportActionsQueue: {
-    context: any;
-    finalItem: any;
-    attributeMappings: AttributeMappings;
-  }[] = [];
+  // private postImportActionsQueue: {
+  //   context: any;
+  //   finalItem: any;
+  //   attributeMappings: AttributeMappings;
+  // }[] = [];
 
   constructor(
     config: AppwriteConfig,
@@ -90,7 +103,7 @@ export class ImportController {
       console.log(`---------------------------------`);
       await this.importCollections(db);
       await resolveAndUpdateRelationships(db.$id, this.database!, this.config!);
-      await this.executePostImportActions();
+      await this.executePostImportActions(db.$id);
       console.log(`---------------------------------`);
       console.log(`Finished import data for database: ${db.name}`);
       console.log(`---------------------------------`);
@@ -188,6 +201,12 @@ export class ImportController {
       );
       await this.processBatch(db, collection, importDef, dataToImport);
     }
+
+    await setAllPendingAfterImportActionsToReady(
+      this.database,
+      db.$id,
+      collection.$id
+    );
   }
 
   async loadData(importDef: ImportDef): Promise<any[]> {
@@ -268,6 +287,7 @@ export class ImportController {
               userToCreate.data
             );
             createIdToUse = user.$id;
+            context.docId = createIdToUse;
             context = { ...context, ...user };
             console.log(
               "Created user, deleting keys in finalItem that exist in user..."
@@ -347,11 +367,26 @@ export class ImportController {
               finalItem
             );
           if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
-            this.postImportActionsQueue.push({
-              context: afterImportActionContext,
+            logger.info(
+              `Pushing to post-import actions queue for ${context.docId}`
+            );
+            const afterImportOperationContext = ContextObject.parse({
+              dbId: db.$id,
+              collectionId: collection.$id,
               finalItem: finalItem,
               attributeMappings: attributeMappingsWithActions,
+              context: afterImportActionContext,
             });
+            await createOrFindAfterImportOperation(
+              this.database,
+              context.collId,
+              afterImportOperationContext
+            );
+            // this.postImportActionsQueue.push({
+            //   context: afterImportActionContext,
+            //   finalItem: finalItem,
+            //   attributeMappings: attributeMappingsWithActions,
+            // });
           }
         })
       );
@@ -480,30 +515,86 @@ export class ImportController {
     });
   }
 
-  async executePostImportActions() {
-    const results = await Promise.allSettled(
-      this.postImportActionsQueue.map(async (action) => {
-        const { context, finalItem, attributeMappings } = action;
-        console.log(
-          `Executing post-import actions for document: ${context.docId}`
-        );
-        return this.importDataActions.executeAfterImportActions(
-          finalItem,
-          attributeMappings,
-          context
-        );
-      })
-    );
-
+  async executePostImportActions(dbId: string) {
+    const collectionActionsPromises = [];
+    for (const collection of this.config.collections) {
+      collectionActionsPromises.push(
+        this.executeActionsInParallel(dbId, collection)
+      );
+    }
+    const results = await Promise.allSettled(collectionActionsPromises);
     results.forEach((result) => {
       if (result.status === "rejected") {
-        console.error(
-          "A post-import action promise was rejected:",
-          result.reason
-        );
+        console.error("A process batch promise was rejected:", result.reason);
       }
     });
+  }
 
-    this.postImportActionsQueue = [];
+  async executeActionsInParallel(dbId: string, collection: ConfigCollection) {
+    const collectionExists = await checkForCollection(
+      this.database,
+      dbId,
+      collection
+    );
+    if (!collectionExists) {
+      logger.error(`No collection found for ${collection.name}`);
+      return; // Skip this iteration
+    }
+    const operations = await getAfterImportOperations(
+      this.database,
+      collectionExists.$id
+    );
+
+    for (const operation of operations) {
+      if (!operation.batches) {
+        continue;
+      }
+      const batches = operation.batches;
+      const promises = [];
+      for (const batch of batches) {
+        const batchId = batch;
+        promises.push(
+          this.database.getDocument("migrations", "batches", batchId)
+        );
+      }
+      const results = await Promise.allSettled(promises);
+      results.forEach((result) => {
+        if (result.status === "rejected") {
+          logger.error("A process batch promise was rejected:", result.reason);
+        }
+      });
+      const resultsData = results
+        .map((result) => (result.status === "fulfilled" ? result.value : null))
+        .filter((result: any) => result !== null && !result.processed)
+        .map((result) => BatchSchema.parse(result));
+      for (const batch of resultsData) {
+        const actionOperation = ContextObject.parse(JSON.parse(batch.data));
+        const { context, finalItem, attributeMappings } = actionOperation;
+        try {
+          await this.importDataActions.executeAfterImportActions(
+            finalItem,
+            attributeMappings,
+            context
+          );
+          // Mark batch as processed
+          await this.database.deleteDocument(
+            "migrations",
+            "batches",
+            batch.$id
+          );
+          await updateOperation(this.database, operation.$id, {
+            status: "completed",
+            batches: [],
+          });
+        } catch (error) {
+          logger.error(`Failed to execute batch ${batch.$id}:`, error);
+        }
+      }
+
+      // After processing all batches, update the operation status
+      await updateOperation(this.database, operation.$id, {
+        status: "completed", // Or determine based on batch success/failure
+      });
+    }
   }
 }

package/src/migrations/migrationHelper.ts

@@ -1,21 +1,182 @@
 import { ID, Query, type Databases } from "node-appwrite";
-import { OperationSchema } from "./backup.js";
+import { BatchSchema, OperationSchema, type Operation } from "./backup.js";
+import { type AttributeMappings, AttributeMappingsSchema } from "./schema.js";
+import { z } from "zod";
+import { logger } from "./logging.js";
+
+/**
+ * Object that contains the context for an action that needs to be executed after import
+ * Used in the afterImportActionsDefinitions
+ * @type {ContextObject}
+ * @typedef {Object} ContextObject
+ * @property {string} collectionId - The ID of the collection
+ * @property {any} finalItem - The final item that was imported
+ * @property {string} action - The name of the action
+ * @property {string[]} params - The parameters for the action
+ * @property {Object} context - The context object for the action (all the data of this specific item)
+ */
+export const ContextObject = z.object({
+  dbId: z.string(),
+  collectionId: z.string(),
+  finalItem: z.any(),
+  attributeMappings: AttributeMappingsSchema,
+  context: z.any(),
+});
+
+export type ContextObject = z.infer<typeof ContextObject>;
+
+export const createOrFindAfterImportOperation = async (
+  database: Databases,
+  collectionId: string,
+  context: ContextObject
+) => {
+  let operation = await findOrCreateOperation(
+    database,
+    collectionId,
+    "afterImportAction"
+  );
+  if (!operation.batches) {
+    operation.batches = [];
+  }
+
+  // Directly create a new batch for the context without checking for an existing batch
+  const contextData = JSON.stringify(context);
+  // Create a new batch with the contextData
+  const newBatchId = await addBatch(database, operation, contextData);
+  // Update the operation with the new batch's $id
+  operation.batches.push(newBatchId);
+  await database.updateDocument(
+    "migrations",
+    "currentOperations",
+    operation.$id,
+    { batches: operation.batches }
+  );
+};
+
+export const addBatch = async (
+  database: Databases,
+  operation: Operation,
+  data: string
+) => {
+  const batch = await database.createDocument(
+    "migrations",
+    "batches",
+    ID.unique(),
+    {
+      data,
+      processed: false,
+    }
+  );
+  await database.updateDocument(
+    "migrations",
+    "currentOperations",
+    operation.$id,
+    {
+      batches: [...(operation.batches || []), batch.$id],
+    }
+  );
+  return batch.$id;
+};
+
+export const getAfterImportOperations = async (
+  database: Databases,
+  collectionId: string
+) => {
+  let lastDocumentId: string | undefined;
+  const allOperations = [];
+  let total = 0;
+
+  do {
+    const query = [
+      Query.equal("collectionId", collectionId),
+      Query.equal("operationType", "afterImportAction"),
+      Query.equal("status", "ready"),
+      Query.limit(100),
+    ];
+
+    if (lastDocumentId) {
+      query.push(Query.cursorAfter(lastDocumentId));
+    }
+
+    const operations = await database.listDocuments(
+      "migrations",
+      "currentOperations",
+      query
+    );
+    total = operations.total; // Update total with the latest fetch
+    allOperations.push(...operations.documents);
+
+    if (operations.documents.length > 0) {
+      lastDocumentId =
+        operations.documents[operations.documents.length - 1].$id;
+    }
+  } while (allOperations.length < total);
+
+  const allOps = allOperations.map((op) => OperationSchema.parse(op));
+  return allOps;
+};
+
+export const setAllPendingAfterImportActionsToReady = async (
+  database: Databases,
+  dbId: string,
+  collectionId: string
+) => {
+  let lastDocumentId: string | undefined;
+  do {
+    const query = [
+      Query.equal("collectionId", collectionId),
+      Query.equal("status", "pending"),
+      Query.limit(100),
+    ];
+
+    if (lastDocumentId) {
+      query.push(Query.cursorAfter(lastDocumentId));
+    }
+
+    const operations = await database.listDocuments(
+      "migrations",
+      "currentOperations",
+      query
+    );
+
+    // Update each pending operation to 'ready'
+    for (const operation of operations.documents) {
+      await database.updateDocument(
+        "migrations",
+        "currentOperations",
+        operation.$id,
+        { status: "ready" }
+      );
+    }
+
+    // Prepare for the next iteration in case there are more than 100 documents
+    if (operations.documents.length > 0) {
+      lastDocumentId =
+        operations.documents[operations.documents.length - 1].$id;
+    } else {
+      lastDocumentId = undefined; // No more documents to process
+    }
+  } while (lastDocumentId); // Continue if there's a last document indicating more documents might exist
+
+  logger.info(
+    `All pending operations for collection ${collectionId} are now set to ready.`
+  );
+};
 
 export const findOrCreateOperation = async (
   database: Databases,
   collectionId: string,
-  operationType: string
+  operationType: string,
+  additionalQueries?: string[]
 ) => {
-  // Here you would query your database for an existing operation
-  // If it doesn't exist, create a new one
-  // This is a simplified example
   const operations = await database.listDocuments(
     "migrations",
     "currentOperations",
     [
       Query.equal("collectionId", collectionId),
       Query.equal("operationType", operationType),
-      Query.equal("status", "in_progress"),
+      Query.equal("status", "pending"),
+      ...(additionalQueries || []),
     ]
   );
 
@@ -30,7 +191,7 @@ export const findOrCreateOperation = async (
       {
         operationType,
         collectionId,
-        status: "in_progress",
+        status: "pending",
         batches: [],
         progress: 0,
         total: 0,