appwrite-utils-cli 0.0.3 → 0.0.5

package/README.md

@@ -77,4 +77,8 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 - Syncing configuration
 - Better file format for config (potentially)
-- Separation of collections and import configuration from main config
+- Separation of collections and import configuration from main config
+
+### Changelog
+
+- 0.0.5: Added `setFieldFromOtherCollectionDocuments` to set an array of ID's for instance from another collection as a `postImportAction`

package/dist/migrations/afterImportActions.d.ts

@@ -7,6 +7,11 @@ export declare const afterImportActions: {
     updateCreatedDocument: (config: AppwriteConfig, dbId: string, collId: string, docId: string, data: any) => Promise<void>;
     checkAndUpdateFieldInDocument: (config: AppwriteConfig, dbId: string, collId: string, docId: string, fieldName: string, oldFieldValue: any, newFieldValue: any) => Promise<void>;
     setFieldFromOtherCollectionDocument: (config: AppwriteConfig, dbId: string, collIdOrName: string, docId: string, fieldName: string, otherCollIdOrName: string, otherDocId: string, otherFieldName: string) => Promise<void>;
+    /**
+     * Updates a field in a document by setting it with document IDs from another collection
+     * based on a matching field value.
+     */
+    setFieldFromOtherCollectionDocuments: (config: AppwriteConfig, dbId: string, collIdOrName: string, docId: string, fieldName: string, otherCollIdOrName: string, matchingFieldName: string, matchingFieldValue: any) => Promise<void>;
     createOrGetBucket: (config: AppwriteConfig, bucketName: string, bucketId?: string, permissions?: string[], fileSecurity?: boolean, enabled?: boolean, maxFileSize?: number, allowedExtensions?: string[], compression?: string, encryption?: boolean, antivirus?: boolean) => Promise<Models.Bucket | undefined>;
     createFileAndUpdateField: (config: AppwriteConfig, dbId: string, collId: string, docId: string, fieldName: string, bucketId: string, filePath: string, fileName: string) => Promise<void>;
 };

package/dist/migrations/afterImportActions.js

@@ -79,6 +79,67 @@ export const afterImportActions = {
             console.error("Error setting field from other collection document: ", error);
         }
     },
+    /**
+     * Updates a field in a document by setting it with document IDs from another collection
+     * based on a matching field value.
+     */
+    setFieldFromOtherCollectionDocuments: async (config, dbId, collIdOrName, docId, fieldName, otherCollIdOrName, matchingFieldName, matchingFieldValue) => {
+        const db = getDatabaseFromConfig(config);
+        // Helper function to find a collection ID by name or return the ID if given
+        const findCollectionId = async (collectionIdentifier) => {
+            const collections = await db.listCollections(dbId, [
+                Query.equal("name", collectionIdentifier),
+                Query.limit(1),
+            ]);
+            return collections.total > 0
+                ? collections.collections[0].$id
+                : collectionIdentifier;
+        };
+        // Function to check if the target field is an array
+        const isTargetFieldArray = async (collectionId, fieldName) => {
+            const collection = await db.getCollection(dbId, collectionId);
+            const attribute = collection.attributes.find((attr) => attr.key === fieldName);
+            // @ts-ignore
+            return attribute?.array === true;
+        };
+        try {
+            const targetCollectionId = await findCollectionId(collIdOrName);
+            const otherCollectionId = await findCollectionId(otherCollIdOrName);
+            const targetFieldIsArray = await isTargetFieldArray(targetCollectionId, fieldName);
+            // Function to recursively fetch all matching documents from the other collection
+            const fetchAllMatchingDocuments = async (cursor) => {
+                const docLimit = 100;
+                const queries = targetFieldIsArray
+                    ? // @ts-ignore
+                        [Query.contains(matchingFieldName, [matchingFieldValue])]
+                    : [Query.equal(matchingFieldName, matchingFieldValue)];
+                if (cursor) {
+                    queries.push(Query.cursorAfter(cursor));
+                }
+                queries.push(Query.limit(docLimit));
+                const response = await db.listDocuments(dbId, otherCollectionId, queries);
+                const documents = response.documents;
+                if (documents.length === 0 || documents.length < docLimit) {
+                    return documents;
+                }
+                const nextCursor = documents[documents.length - 1].$id;
+                const nextBatch = await fetchAllMatchingDocuments(nextCursor);
+                return documents.concat(nextBatch);
+            };
+            const matchingDocuments = await fetchAllMatchingDocuments();
+            const documentIds = matchingDocuments.map((doc) => doc.$id);
+            if (documentIds.length > 0) {
+                const updatePayload = targetFieldIsArray
+                    ? { [fieldName]: documentIds }
+                    : { [fieldName]: documentIds[0] };
+                await db.updateDocument(dbId, targetCollectionId, docId, updatePayload);
+                console.log(`Field ${fieldName} updated successfully in document ${docId} with ${documentIds.length} document IDs.`);
+            }
+        }
+        catch (error) {
+            console.error("Error setting field from other collection documents: ", error);
+        }
+    },
     createOrGetBucket: async (config, bucketName, bucketId, permissions, fileSecurity, enabled, maxFileSize, allowedExtensions, compression, encryption, antivirus) => {
         try {
             const storage = getStorageFromConfig(config);
@@ -157,6 +218,9 @@ export const afterImportActions = {
                 else {
                     updateData = file.$id; // Set the new file ID
                 }
+                await db.updateDocument(dbId, collId, doc.$id, {
+                    [fieldName]: updateData,
+                });
                 // console.log(
                 //   "Updating document with file: ",
                 //   doc.$id,

package/dist/migrations/backup.d.ts

@@ -114,10 +114,10 @@ export declare const OperationSchema: z.ZodObject<{
     progress: z.ZodNumber;
     total: z.ZodNumber;
     error: z.ZodString;
-    status: z.ZodEnum<["pending", "in_progress", "completed", "error"]>;
+    status: z.ZodDefault<z.ZodEnum<["pending", "ready", "in_progress", "completed", "error", "cancelled"]>>;
 }, "strip", z.ZodTypeAny, {
     error: string;
-    status: "error" | "pending" | "in_progress" | "completed";
+    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
     $id: string;
     $createdAt: string;
     $updatedAt: string;
@@ -129,7 +129,6 @@ export declare const OperationSchema: z.ZodObject<{
     batches?: string[] | undefined;
 }, {
     error: string;
-    status: "error" | "pending" | "in_progress" | "completed";
     $id: string;
     $createdAt: string;
     $updatedAt: string;
@@ -139,6 +138,7 @@ export declare const OperationSchema: z.ZodObject<{
     total: number;
     data?: any;
     batches?: string[] | undefined;
+    status?: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled" | undefined;
 }>;
 export type Operation = z.infer<typeof OperationSchema>;
 export declare const OperationCreateSchema: z.ZodObject<Omit<{
@@ -152,10 +152,10 @@ export declare const OperationCreateSchema: z.ZodObject<Omit<{
     progress: z.ZodNumber;
     total: z.ZodNumber;
     error: z.ZodString;
-    status: z.ZodEnum<["pending", "in_progress", "completed", "error"]>;
+    status: z.ZodDefault<z.ZodEnum<["pending", "ready", "in_progress", "completed", "error", "cancelled"]>>;
 }, "$id" | "$createdAt" | "$updatedAt">, "strip", z.ZodTypeAny, {
     error: string;
-    status: "error" | "pending" | "in_progress" | "completed";
+    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
     collectionId: string;
     operationType: string;
     progress: number;
@@ -164,11 +164,11 @@ export declare const OperationCreateSchema: z.ZodObject<Omit<{
     batches?: string[] | undefined;
 }, {
     error: string;
-    status: "error" | "pending" | "in_progress" | "completed";
     collectionId: string;
     operationType: string;
     progress: number;
     total: number;
+    status?: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled" | undefined;
     data?: any;
     batches?: string[] | undefined;
 }>;
@@ -263,12 +263,12 @@ export declare const getMigrationCollectionSchemas: () => {
                     targetField?: string | undefined;
                 } | undefined;
             })[];
+            name: string;
             $id: string;
             $permissions: {
                 permission: string;
                 target: string;
             }[];
-            name: string;
             enabled: boolean;
             documentSecurity: boolean;
             indexes: {
@@ -485,12 +485,12 @@ export declare const getMigrationCollectionSchemas: () => {
                     targetField?: string | undefined;
                 } | undefined;
             })[];
+            name: string;
             $id: string;
             $permissions: {
                 permission: string;
                 target: string;
             }[];
-            name: string;
             enabled: boolean;
             documentSecurity: boolean;
             indexes: {

package/dist/migrations/backup.js

@@ -44,7 +44,16 @@ export const OperationSchema = z.object({
     progress: z.number(),
     total: z.number(),
     error: z.string(),
-    status: z.enum(["pending", "in_progress", "completed", "error"]),
+    status: z
+        .enum([
+        "pending",
+        "ready",
+        "in_progress",
+        "completed",
+        "error",
+        "cancelled",
+    ])
+        .default("pending"),
 });
 export const OperationCreateSchema = OperationSchema.omit({
     $id: true,
@@ -107,7 +116,14 @@ export const getMigrationCollectionSchemas = () => {
         attributeSchema.parse({
             key: "status",
             type: "enum",
-            elements: ["pending", "in_progress", "completed", "error"],
+            elements: [
+                "pending",
+                "ready",
+                "in_progress",
+                "completed",
+                "error",
+                "cancelled",
+            ],
             error: "Invalid Status",
             array: false,
             xdefault: "pending",

package/dist/migrations/importController.d.ts

@@ -11,7 +11,6 @@ export declare class ImportController {
     private setupOptions;
     private documentCache;
     private batchLimit;
-    private postImportActionsQueue;
     constructor(config: AppwriteConfig, database: Databases, storage: Storage, appwriteFolderPath: string, importDataActions: ImportDataActions, setupOptions: SetupOptions);
     run(): Promise<void>;
     importCollections(db: ConfigDatabase): Promise<void>;
@@ -40,5 +39,6 @@ export declare class ImportController {
             name: string;
         } | undefined;
     }[];
-    executePostImportActions(): Promise<void>;
+    executePostImportActions(dbId: string): Promise<void>;
+    executeActionsInParallel(dbId: string, collection: ConfigCollection): Promise<void>;
 }

package/dist/migrations/importController.js

@@ -9,6 +9,9 @@ import { areCollectionNamesSame } from "../utils/index.js";
 import { resolveAndUpdateRelationships } from "./relationships.js";
 import { AuthUserCreateSchema } from "../types.js";
 import { UsersController } from "./users.js";
+import { logger } from "./logging.js";
+import { ContextObject, createOrFindAfterImportOperation, getAfterImportOperations, setAllPendingAfterImportActionsToReady, updateOperation, } from "./migrationHelper.js";
+import { BatchSchema, OperationCreateSchema, OperationSchema, } from "./backup.js";
 export class ImportController {
     config;
     database;
@@ -18,7 +21,11 @@ export class ImportController {
     setupOptions;
     documentCache;
     batchLimit = 25; // Define batch size limit
-    postImportActionsQueue = [];
+    // private postImportActionsQueue: {
+    //   context: any;
+    //   finalItem: any;
+    //   attributeMappings: AttributeMappings;
+    // }[] = [];
     constructor(config, database, storage, appwriteFolderPath, importDataActions, setupOptions) {
         this.config = config;
         this.database = database;
@@ -55,7 +62,7 @@ export class ImportController {
             console.log(`---------------------------------`);
             await this.importCollections(db);
             await resolveAndUpdateRelationships(db.$id, this.database, this.config);
-            await this.executePostImportActions();
+            await this.executePostImportActions(db.$id);
             console.log(`---------------------------------`);
             console.log(`Finished import data for database: ${db.name}`);
             console.log(`---------------------------------`);
@@ -113,6 +120,7 @@ export class ImportController {
             console.log(`Processing update definitions for collection ID: ${collection.$id}`);
             await this.processBatch(db, collection, importDef, dataToImport);
         }
+        await setAllPendingAfterImportActionsToReady(this.database, db.$id, collection.$id);
     }
     async loadData(importDef) {
         const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
@@ -161,6 +169,7 @@ export class ImportController {
                     }
                     const user = await usersController.createUserAndReturn(userToCreate.data);
                     createIdToUse = user.$id;
+                    context.docId = createIdToUse;
                     context = { ...context, ...user };
                     console.log("Created user, deleting keys in finalItem that exist in user...");
                     const associatedDocFound = await this.database.listDocuments(db.$id, context.collId, [Query.equal("$id", createIdToUse)]);
@@ -205,11 +214,20 @@ export class ImportController {
                 const afterImportActionContext = structuredClone(context);
                 const attributeMappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, context, finalItem);
                 if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
-                    this.postImportActionsQueue.push({
-                        context: afterImportActionContext,
+                    logger.info(`Pushing to post-import actions queue for ${context.docId}`);
+                    const afterImportOperationContext = ContextObject.parse({
+                        dbId: db.$id,
+                        collectionId: collection.$id,
                         finalItem: finalItem,
                         attributeMappings: attributeMappingsWithActions,
+                        context: afterImportActionContext,
                     });
+                    await createOrFindAfterImportOperation(this.database, context.collId, afterImportOperationContext);
+                    // this.postImportActionsQueue.push({
+                    //   context: afterImportActionContext,
+                    //   finalItem: finalItem,
+                    //   attributeMappings: attributeMappingsWithActions,
+                    // });
                 }
             }));
             results.forEach((result) => {
@@ -296,17 +314,65 @@ export class ImportController {
             return mapping;
         });
     }
-    async executePostImportActions() {
-        const results = await Promise.allSettled(this.postImportActionsQueue.map(async (action) => {
-            const { context, finalItem, attributeMappings } = action;
-            console.log(`Executing post-import actions for document: ${context.docId}`);
-            return this.importDataActions.executeAfterImportActions(finalItem, attributeMappings, context);
-        }));
+    async executePostImportActions(dbId) {
+        const collectionActionsPromises = [];
+        for (const collection of this.config.collections) {
+            collectionActionsPromises.push(this.executeActionsInParallel(dbId, collection));
+        }
+        const results = await Promise.allSettled(collectionActionsPromises);
         results.forEach((result) => {
             if (result.status === "rejected") {
-                console.error("A post-import action promise was rejected:", result.reason);
+                console.error("A process batch promise was rejected:", result.reason);
             }
         });
-        this.postImportActionsQueue = [];
+    }
+    async executeActionsInParallel(dbId, collection) {
+        const collectionExists = await checkForCollection(this.database, dbId, collection);
+        if (!collectionExists) {
+            logger.error(`No collection found for ${collection.name}`);
+            return; // Skip this iteration
+        }
+        const operations = await getAfterImportOperations(this.database, collectionExists.$id);
+        for (const operation of operations) {
+            if (!operation.batches) {
+                continue;
+            }
+            const batches = operation.batches;
+            const promises = [];
+            for (const batch of batches) {
+                const batchId = batch;
+                promises.push(this.database.getDocument("migrations", "batches", batchId));
+            }
+            const results = await Promise.allSettled(promises);
+            results.forEach((result) => {
+                if (result.status === "rejected") {
+                    logger.error("A process batch promise was rejected:", result.reason);
+                }
+            });
+            const resultsData = results
+                .map((result) => (result.status === "fulfilled" ? result.value : null))
+                .filter((result) => result !== null && !result.processed)
+                .map((result) => BatchSchema.parse(result));
+            for (const batch of resultsData) {
+                const actionOperation = ContextObject.parse(JSON.parse(batch.data));
+                const { context, finalItem, attributeMappings } = actionOperation;
+                try {
+                    await this.importDataActions.executeAfterImportActions(finalItem, attributeMappings, context);
+                    // Mark batch as processed
+                    await this.database.deleteDocument("migrations", "batches", batch.$id);
+                    await updateOperation(this.database, operation.$id, {
+                        status: "completed",
+                        batches: [],
+                    });
+                }
+                catch (error) {
+                    logger.error(`Failed to execute batch ${batch.$id}:`, error);
+                }
+            }
+            // After processing all batches, update the operation status
+            await updateOperation(this.database, operation.$id, {
+                status: "completed", // Or determine based on batch success/failure
+            });
+        }
     }
 }

package/dist/migrations/migrationHelper.d.ts

@@ -1,7 +1,159 @@
 import { type Databases } from "node-appwrite";
-export declare const findOrCreateOperation: (database: Databases, collectionId: string, operationType: string) => Promise<{
+import { type Operation } from "./backup.js";
+import { z } from "zod";
+/**
+ * Object that contains the context for an action that needs to be executed after import
+ * Used in the afterImportActionsDefinitions
+ * @type {ContextObject}
+ * @typedef {Object} ContextObject
+ * @property {string} collectionId - The ID of the collection
+ * @property {any} finalItem - The final item that was imported
+ * @property {string} action - The name of the action
+ * @property {string[]} params - The parameters for the action
+ * @property {Object} context - The context object for the action (all the data of this specific item)
+ */
+export declare const ContextObject: z.ZodObject<{
+    dbId: z.ZodString;
+    collectionId: z.ZodString;
+    finalItem: z.ZodAny;
+    attributeMappings: z.ZodArray<z.ZodObject<{
+        oldKey: z.ZodOptional<z.ZodString>;
+        oldKeys: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        targetKey: z.ZodString;
+        fileData: z.ZodOptional<z.ZodObject<{
+            name: z.ZodString;
+            path: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            path: string;
+            name: string;
+        }, {
+            path: string;
+            name: string;
+        }>>;
+        converters: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+        validationActions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            action: z.ZodString;
+            params: z.ZodArray<z.ZodString, "many">;
+        }, "strip", z.ZodTypeAny, {
+            params: string[];
+            action: string;
+        }, {
+            params: string[];
+            action: string;
+        }>, "many">>;
+        postImportActions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            action: z.ZodString;
+            params: z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodAny>]>, "many">;
+        }, "strip", z.ZodTypeAny, {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }, {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }>, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        targetKey: string;
+        converters: string[];
+        validationActions: {
+            params: string[];
+            action: string;
+        }[];
+        postImportActions: {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }[];
+        oldKey?: string | undefined;
+        oldKeys?: string[] | undefined;
+        fileData?: {
+            path: string;
+            name: string;
+        } | undefined;
+    }, {
+        targetKey: string;
+        oldKey?: string | undefined;
+        oldKeys?: string[] | undefined;
+        fileData?: {
+            path: string;
+            name: string;
+        } | undefined;
+        converters?: string[] | undefined;
+        validationActions?: {
+            params: string[];
+            action: string;
+        }[] | undefined;
+        postImportActions?: {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }[] | undefined;
+    }>, "many">;
+    context: z.ZodAny;
+}, "strip", z.ZodTypeAny, {
+    attributeMappings: {
+        targetKey: string;
+        converters: string[];
+        validationActions: {
+            params: string[];
+            action: string;
+        }[];
+        postImportActions: {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }[];
+        oldKey?: string | undefined;
+        oldKeys?: string[] | undefined;
+        fileData?: {
+            path: string;
+            name: string;
+        } | undefined;
+    }[];
+    collectionId: string;
+    dbId: string;
+    finalItem?: any;
+    context?: any;
+}, {
+    attributeMappings: {
+        targetKey: string;
+        oldKey?: string | undefined;
+        oldKeys?: string[] | undefined;
+        fileData?: {
+            path: string;
+            name: string;
+        } | undefined;
+        converters?: string[] | undefined;
+        validationActions?: {
+            params: string[];
+            action: string;
+        }[] | undefined;
+        postImportActions?: {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }[] | undefined;
+    }[];
+    collectionId: string;
+    dbId: string;
+    finalItem?: any;
+    context?: any;
+}>;
+export type ContextObject = z.infer<typeof ContextObject>;
+export declare const createOrFindAfterImportOperation: (database: Databases, collectionId: string, context: ContextObject) => Promise<void>;
+export declare const addBatch: (database: Databases, operation: Operation, data: string) => Promise<string>;
+export declare const getAfterImportOperations: (database: Databases, collectionId: string) => Promise<{
+    error: string;
+    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
+    $id: string;
+    $createdAt: string;
+    $updatedAt: string;
+    collectionId: string;
+    operationType: string;
+    progress: number;
+    total: number;
+    data?: any;
+    batches?: string[] | undefined;
+}[]>;
+export declare const setAllPendingAfterImportActionsToReady: (database: Databases, dbId: string, collectionId: string) => Promise<void>;
+export declare const findOrCreateOperation: (database: Databases, collectionId: string, operationType: string, additionalQueries?: string[]) => Promise<{
     error: string;
-    status: "error" | "pending" | "in_progress" | "completed";
+    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
     $id: string;
     $createdAt: string;
     $updatedAt: string;

package/dist/migrations/migrationHelper.js

@@ -1,13 +1,107 @@
 import { ID, Query } from "node-appwrite";
-import { OperationSchema } from "./backup.js";
-export const findOrCreateOperation = async (database, collectionId, operationType) => {
-    // Here you would query your database for an existing operation
-    // If it doesn't exist, create a new one
-    // This is a simplified example
+import { BatchSchema, OperationSchema } from "./backup.js";
+import { AttributeMappingsSchema } from "./schema.js";
+import { z } from "zod";
+import { logger } from "./logging.js";
+/**
+ * Object that contains the context for an action that needs to be executed after import
+ * Used in the afterImportActionsDefinitions
+ * @type {ContextObject}
+ * @typedef {Object} ContextObject
+ * @property {string} collectionId - The ID of the collection
+ * @property {any} finalItem - The final item that was imported
+ * @property {string} action - The name of the action
+ * @property {string[]} params - The parameters for the action
+ * @property {Object} context - The context object for the action (all the data of this specific item)
+ */
+export const ContextObject = z.object({
+    dbId: z.string(),
+    collectionId: z.string(),
+    finalItem: z.any(),
+    attributeMappings: AttributeMappingsSchema,
+    context: z.any(),
+});
+export const createOrFindAfterImportOperation = async (database, collectionId, context) => {
+    let operation = await findOrCreateOperation(database, collectionId, "afterImportAction");
+    if (!operation.batches) {
+        operation.batches = [];
+    }
+    // Directly create a new batch for the context without checking for an existing batch
+    const contextData = JSON.stringify(context);
+    // Create a new batch with the contextData
+    const newBatchId = await addBatch(database, operation, contextData);
+    // Update the operation with the new batch's $id
+    operation.batches.push(newBatchId);
+    await database.updateDocument("migrations", "currentOperations", operation.$id, { batches: operation.batches });
+};
+export const addBatch = async (database, operation, data) => {
+    const batch = await database.createDocument("migrations", "batches", ID.unique(), {
+        data,
+        processed: false,
+    });
+    await database.updateDocument("migrations", "currentOperations", operation.$id, {
+        batches: [...(operation.batches || []), batch.$id],
+    });
+    return batch.$id;
+};
+export const getAfterImportOperations = async (database, collectionId) => {
+    let lastDocumentId;
+    const allOperations = [];
+    let total = 0;
+    do {
+        const query = [
+            Query.equal("collectionId", collectionId),
+            Query.equal("operationType", "afterImportAction"),
+            Query.equal("status", "ready"),
+            Query.limit(100),
+        ];
+        if (lastDocumentId) {
+            query.push(Query.cursorAfter(lastDocumentId));
+        }
+        const operations = await database.listDocuments("migrations", "currentOperations", query);
+        total = operations.total; // Update total with the latest fetch
+        allOperations.push(...operations.documents);
+        if (operations.documents.length > 0) {
+            lastDocumentId =
+                operations.documents[operations.documents.length - 1].$id;
+        }
+    } while (allOperations.length < total);
+    const allOps = allOperations.map((op) => OperationSchema.parse(op));
+    return allOps;
+};
+export const setAllPendingAfterImportActionsToReady = async (database, dbId, collectionId) => {
+    let lastDocumentId;
+    do {
+        const query = [
+            Query.equal("collectionId", collectionId),
+            Query.equal("status", "pending"),
+            Query.limit(100),
+        ];
+        if (lastDocumentId) {
+            query.push(Query.cursorAfter(lastDocumentId));
+        }
+        const operations = await database.listDocuments("migrations", "currentOperations", query);
+        // Update each pending operation to 'ready'
+        for (const operation of operations.documents) {
+            await database.updateDocument("migrations", "currentOperations", operation.$id, { status: "ready" });
+        }
+        // Prepare for the next iteration in case there are more than 100 documents
+        if (operations.documents.length > 0) {
+            lastDocumentId =
+                operations.documents[operations.documents.length - 1].$id;
+        }
+        else {
+            lastDocumentId = undefined; // No more documents to process
+        }
+    } while (lastDocumentId); // Continue if there's a last document indicating more documents might exist
+    logger.info(`All pending operations for collection ${collectionId} are now set to ready.`);
+};
+export const findOrCreateOperation = async (database, collectionId, operationType, additionalQueries) => {
     const operations = await database.listDocuments("migrations", "currentOperations", [
         Query.equal("collectionId", collectionId),
         Query.equal("operationType", operationType),
-        Query.equal("status", "in_progress"),
+        Query.equal("status", "pending"),
+        ...(additionalQueries || []),
     ]);
     if (operations.documents.length > 0) {
         return OperationSchema.parse(operations.documents[0]); // Assuming the first document is the operation we want
@@ -17,7 +111,7 @@ export const findOrCreateOperation = async (database, collectionId, operationTyp
         const op = await database.createDocument("migrations", "currentOperations", ID.unique(), {
             operationType,
             collectionId,
-            status: "in_progress",
+            status: "pending",
             batches: [],
             progress: 0,
             total: 0,