appwrite-utils-cli 0.10.85 → 1.0.1

package/dist/shared/migrationHelpers.d.ts

@@ -0,0 +1,173 @@
+import { type Databases } from "node-appwrite";
+import { z } from "zod";
+/**
+ * Object that contains the context for an action that needs to be executed after import
+ * Used in the afterImportActionsDefinitions
+ * @type {ContextObject}
+ * @typedef {Object} ContextObject
+ * @property {string} collectionId - The ID of the collection
+ * @property {any} finalItem - The final item that was imported
+ * @property {string} action - The name of the action
+ * @property {string[]} params - The parameters for the action
+ * @property {Object} context - The context object for the action (all the data of this specific item)
+ */
+export declare const ContextObject: z.ZodObject<{
+    dbId: z.ZodString;
+    collectionId: z.ZodString;
+    finalItem: z.ZodAny;
+    attributeMappings: z.ZodArray<z.ZodObject<{
+        oldKey: z.ZodOptional<z.ZodString>;
+        oldKeys: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        targetKey: z.ZodString;
+        valueToSet: z.ZodOptional<z.ZodAny>;
+        fileData: z.ZodOptional<z.ZodObject<{
+            name: z.ZodString;
+            path: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            path: string;
+            name: string;
+        }, {
+            path: string;
+            name: string;
+        }>>;
+        converters: z.ZodOptional<z.ZodDefault<z.ZodArray<z.ZodString, "many">>>;
+        validationActions: z.ZodOptional<z.ZodDefault<z.ZodArray<z.ZodObject<{
+            action: z.ZodString;
+            params: z.ZodArray<z.ZodString, "many">;
+        }, "strip", z.ZodTypeAny, {
+            params: string[];
+            action: string;
+        }, {
+            params: string[];
+            action: string;
+        }>, "many">>>;
+        postImportActions: z.ZodOptional<z.ZodDefault<z.ZodArray<z.ZodObject<{
+            action: z.ZodString;
+            params: z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodAny>]>, "many">;
+        }, "strip", z.ZodTypeAny, {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }, {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }>, "many">>>;
+    }, "strip", z.ZodTypeAny, {
+        targetKey: string;
+        oldKey?: string | undefined;
+        oldKeys?: string[] | undefined;
+        valueToSet?: any;
+        fileData?: {
+            path: string;
+            name: string;
+        } | undefined;
+        converters?: string[] | undefined;
+        validationActions?: {
+            params: string[];
+            action: string;
+        }[] | undefined;
+        postImportActions?: {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }[] | undefined;
+    }, {
+        targetKey: string;
+        oldKey?: string | undefined;
+        oldKeys?: string[] | undefined;
+        valueToSet?: any;
+        fileData?: {
+            path: string;
+            name: string;
+        } | undefined;
+        converters?: string[] | undefined;
+        validationActions?: {
+            params: string[];
+            action: string;
+        }[] | undefined;
+        postImportActions?: {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }[] | undefined;
+    }>, "many">;
+    context: z.ZodAny;
+}, "strip", z.ZodTypeAny, {
+    collectionId: string;
+    attributeMappings: {
+        targetKey: string;
+        oldKey?: string | undefined;
+        oldKeys?: string[] | undefined;
+        valueToSet?: any;
+        fileData?: {
+            path: string;
+            name: string;
+        } | undefined;
+        converters?: string[] | undefined;
+        validationActions?: {
+            params: string[];
+            action: string;
+        }[] | undefined;
+        postImportActions?: {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }[] | undefined;
+    }[];
+    dbId: string;
+    context?: any;
+    finalItem?: any;
+}, {
+    collectionId: string;
+    attributeMappings: {
+        targetKey: string;
+        oldKey?: string | undefined;
+        oldKeys?: string[] | undefined;
+        valueToSet?: any;
+        fileData?: {
+            path: string;
+            name: string;
+        } | undefined;
+        converters?: string[] | undefined;
+        validationActions?: {
+            params: string[];
+            action: string;
+        }[] | undefined;
+        postImportActions?: {
+            params: (string | Record<string, any>)[];
+            action: string;
+        }[] | undefined;
+    }[];
+    dbId: string;
+    context?: any;
+    finalItem?: any;
+}>;
+export type ContextObject = z.infer<typeof ContextObject>;
+export declare const createOrFindAfterImportOperation: (database: Databases, collectionId: string, context: ContextObject, useMigrations?: boolean) => Promise<void>;
+export declare const addBatch: (database: Databases, data: string) => Promise<string>;
+export declare const getAfterImportOperations: (database: Databases, collectionId: string, useMigrations?: boolean) => Promise<{
+    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
+    error: string;
+    $id: string;
+    $createdAt: string;
+    $updatedAt: string;
+    collectionId: string;
+    operationType: string;
+    progress: number;
+    total: number;
+    data?: any;
+    batches?: string[] | undefined;
+}[]>;
+export declare const findOrCreateOperation: (database: Databases, collectionId: string, operationType: string, additionalQueries?: string[]) => Promise<{
+    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
+    error: string;
+    $id: string;
+    $createdAt: string;
+    $updatedAt: string;
+    collectionId: string;
+    operationType: string;
+    progress: number;
+    total: number;
+    data?: any;
+    batches?: string[] | undefined;
+}>;
+export declare const updateOperation: (database: Databases, operationId: string, updateFields: any, useMigrations?: boolean) => Promise<void>;
+export declare const maxDataLength = 1073741820;
+export declare const maxBatchItems = 25;
+export declare const splitIntoBatches: (data: any[]) => any[][];

package/dist/shared/migrationHelpers.js

@@ -0,0 +1,142 @@
+import { ID, Query } from "node-appwrite";
+import { BatchSchema, OperationSchema } from "../storage/schemas.js";
+import { AttributeMappingsSchema } from "appwrite-utils";
+import { z } from "zod";
+import { logger } from "./logging.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+/**
+ * Object that contains the context for an action that needs to be executed after import
+ * Used in the afterImportActionsDefinitions
+ * @type {ContextObject}
+ * @typedef {Object} ContextObject
+ * @property {string} collectionId - The ID of the collection
+ * @property {any} finalItem - The final item that was imported
+ * @property {string} action - The name of the action
+ * @property {string[]} params - The parameters for the action
+ * @property {Object} context - The context object for the action (all the data of this specific item)
+ */
+export const ContextObject = z.object({
+    dbId: z.string(),
+    collectionId: z.string(),
+    finalItem: z.any(),
+    attributeMappings: AttributeMappingsSchema,
+    context: z.any(),
+});
+export const createOrFindAfterImportOperation = async (database, collectionId, context, useMigrations = true) => {
+    if (!useMigrations) {
+        logger.info("Migrations disabled, skipping after import operation tracking");
+        return;
+    }
+    let operation = await findOrCreateOperation(database, collectionId, "afterImportAction");
+    if (!operation.batches) {
+        operation.batches = [];
+    }
+    // Directly create a new batch for the context without checking for an existing batch
+    const contextData = JSON.stringify(context);
+    // Create a new batch with the contextData
+    const newBatchId = await addBatch(database, contextData);
+    // Update the operation with the new batch's $id
+    operation.batches = [...operation.batches, newBatchId];
+    await database.updateDocument("migrations", "currentOperations", operation.$id, { batches: operation.batches });
+};
+export const addBatch = async (database, data) => {
+    const batch = await database.createDocument("migrations", "batches", ID.unique(), {
+        data,
+        processed: false,
+    });
+    return batch.$id;
+};
+export const getAfterImportOperations = async (database, collectionId, useMigrations = true) => {
+    if (!useMigrations) {
+        logger.info("Migrations disabled, returning empty operations list");
+        return [];
+    }
+    let lastDocumentId;
+    const allOperations = [];
+    let total = 0;
+    do {
+        const query = [
+            Query.equal("collectionId", collectionId),
+            Query.equal("operationType", "afterImportAction"),
+            Query.limit(100),
+        ];
+        if (lastDocumentId) {
+            query.push(Query.cursorAfter(lastDocumentId));
+        }
+        const operations = await database.listDocuments("migrations", "currentOperations", query);
+        total = operations.total; // Update total with the latest fetch
+        allOperations.push(...operations.documents);
+        if (operations.documents.length > 0 && operations.documents.length >= 100) {
+            lastDocumentId =
+                operations.documents[operations.documents.length - 1].$id;
+        }
+    } while (allOperations.length < total);
+    const allOps = allOperations.map((op) => OperationSchema.parse(op));
+    return allOps;
+};
+export const findOrCreateOperation = async (database, collectionId, operationType, additionalQueries) => {
+    const operations = await tryAwaitWithRetry(async () => await database.listDocuments("migrations", "currentOperations", [
+        Query.equal("collectionId", collectionId),
+        Query.equal("operationType", operationType),
+        Query.equal("status", "pending"),
+        ...(additionalQueries || []),
+    ]));
+    if (operations.documents.length > 0) {
+        return OperationSchema.parse(operations.documents[0]); // Assuming the first document is the operation we want
+    }
+    else {
+        // Create a new operation document
+        const op = await tryAwaitWithRetry(async () => await database.createDocument("migrations", "currentOperations", ID.unique(), {
+            operationType,
+            collectionId,
+            status: "pending",
+            batches: [],
+            progress: 0,
+            total: 0,
+            error: "",
+        }));
+        return OperationSchema.parse(op);
+    }
+};
+export const updateOperation = async (database, operationId, updateFields, useMigrations = true) => {
+    if (!useMigrations) {
+        logger.info("Migrations disabled, skipping operation update");
+        return;
+    }
+    await tryAwaitWithRetry(async () => await database.updateDocument("migrations", "currentOperations", operationId, updateFields));
+};
+// Actual max 1073741824
+export const maxDataLength = 1073741820;
+export const maxBatchItems = 25;
+export const splitIntoBatches = (data) => {
+    let batches = [];
+    let currentBatch = [];
+    let currentBatchLength = 0;
+    let currentBatchItemCount = 0;
+    data.forEach((item, index) => {
+        const itemLength = JSON.stringify(item).length;
+        if (itemLength > maxDataLength) {
+            console.log(item, `Large item found at index ${index} with length ${itemLength}:`);
+        }
+        // Check if adding the current item would exceed the max length or max items per batch
+        if (currentBatchLength + itemLength >= maxDataLength ||
+            currentBatchItemCount >= maxBatchItems) {
+            // If so, start a new batch
+            batches.push(currentBatch);
+            currentBatch = [item];
+            currentBatchLength = itemLength;
+            currentBatchItemCount = 1; // Reset item count for the new batch
+        }
+        else {
+            // Otherwise, add the item to the current batch
+            currentBatch.push(item);
+            currentBatchLength += itemLength;
+            currentBatchItemCount++;
+        }
+    });
+    // Don't forget to add the last batch if it's not empty
+    if (currentBatch.length > 0) {
+        batches.push(currentBatch);
+    }
+    return batches;
+};

package/dist/shared/operationLogger.d.ts

@@ -0,0 +1,3 @@
+import type { Databases, Models } from "node-appwrite";
+import type { OperationCreate } from "../storage/schemas.js";
+export declare const logOperation: (db: Databases, dbId: string, operationDetails: OperationCreate, operationId?: string, useMigrations?: boolean) => Promise<Models.Document | null>;

package/dist/shared/operationLogger.js

@@ -0,0 +1,25 @@
+import { tryAwaitWithRetry } from "appwrite-utils";
+import { ulid } from "ulidx";
+export const logOperation = async (db, dbId, operationDetails, operationId, useMigrations = true) => {
+    if (!useMigrations) {
+        console.log("Migrations disabled, skipping operation logging");
+        return null;
+    }
+    try {
+        let operation;
+        if (operationId) {
+            // Update existing operation log
+            operation = await tryAwaitWithRetry(async () => await db.updateDocument("migrations", "currentOperations", operationId, operationDetails));
+        }
+        else {
+            // Create new operation log
+            operation = await db.createDocument("migrations", "currentOperations", ulid(), operationDetails);
+        }
+        console.log(`Operation logged: ${operation.$id}`);
+        return operation;
+    }
+    catch (error) {
+        console.error(`Error logging operation: ${error}`);
+        throw error;
+    }
+};

package/dist/shared/operationQueue.d.ts

@@ -0,0 +1,13 @@
+import { type Databases, type Models } from "node-appwrite";
+import type { Attribute } from "appwrite-utils";
+export interface QueuedOperation {
+    type: "attribute";
+    collectionId?: string;
+    attribute?: Attribute;
+    collection?: Models.Collection;
+    dependencies?: string[];
+}
+export declare const queuedOperations: QueuedOperation[];
+export declare const nameToIdMapping: Map<string, string>;
+export declare const enqueueOperation: (operation: QueuedOperation) => void;
+export declare const processQueue: (db: Databases, dbId: string) => Promise<void>;

package/dist/shared/operationQueue.js

@@ -0,0 +1,79 @@
+import { Query } from "node-appwrite";
+import { createOrUpdateAttribute } from "../collections/attributes.js";
+import { fetchAndCacheCollectionByName } from "../collections/methods.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+export const queuedOperations = [];
+export const nameToIdMapping = new Map();
+export const enqueueOperation = (operation) => {
+    queuedOperations.push(operation);
+};
+export const processQueue = async (db, dbId) => {
+    console.log("---------------------------------");
+    console.log(`Starting Queue processing of ${dbId}`);
+    console.log("---------------------------------");
+    let progress = true;
+    while (progress) {
+        progress = false;
+        console.log("Processing queued operations:");
+        for (let i = 0; i < queuedOperations.length; i++) {
+            const operation = queuedOperations[i];
+            let collectionFound;
+            // Handle relationship attribute operations
+            if (operation.attribute?.type === "relationship") {
+                // Attempt to resolve the collection directly if collectionId is specified
+                if (operation.collectionId) {
+                    console.log(`\tFetching collection by ID: ${operation.collectionId}`);
+                    try {
+                        collectionFound = await tryAwaitWithRetry(async () => await db.getCollection(dbId, operation.collectionId));
+                    }
+                    catch (e) {
+                        console.log(`\tCollection not found by ID: ${operation.collectionId}`);
+                    }
+                }
+                // Attempt to resolve related collection if specified and not already found
+                if (!collectionFound && operation.attribute?.relatedCollection) {
+                    collectionFound = await fetchAndCacheCollectionByName(db, dbId, operation.attribute.relatedCollection);
+                }
+                // Handle dependencies if collection still not found
+                if (!collectionFound) {
+                    for (const dep of operation.dependencies || []) {
+                        collectionFound = await fetchAndCacheCollectionByName(db, dbId, dep);
+                        if (collectionFound)
+                            break; // Break early if collection is found
+                    }
+                }
+            }
+            else if (operation.collectionId) {
+                // Handle non-relationship operations with a specified collectionId
+                console.log(`\tFetching collection by ID: ${operation.collectionId}`);
+                try {
+                    collectionFound = await tryAwaitWithRetry(async () => await db.getCollection(dbId, operation.collectionId));
+                }
+                catch (e) {
+                    console.log(`\tCollection not found by ID: ${operation.collectionId}`);
+                }
+            }
+            // Process the operation if the collection is found
+            if (collectionFound && operation.attribute) {
+                console.log(`\tProcessing attribute: ${operation.attribute.key} for collection ID: ${collectionFound.$id}`);
+                await createOrUpdateAttribute(db, dbId, collectionFound, operation.attribute);
+                queuedOperations.splice(i, 1);
+                i--; // Adjust index since we're modifying the array
+                progress = true;
+            }
+            else {
+                console.error(`\tCollection not found for operation, removing from queue: ${JSON.stringify(operation)}`);
+                queuedOperations.splice(i, 1);
+                i--; // Adjust index since we're modifying the array
+            }
+        }
+        console.log(`\tFinished processing queued operations`);
+    }
+    if (queuedOperations.length > 0) {
+        console.error("Unresolved operations remain due to unmet dependencies.");
+        console.log(queuedOperations);
+    }
+    console.log("---------------------------------");
+    console.log(`Queue processing complete for ${dbId}`);
+    console.log("---------------------------------");
+};

package/dist/shared/progressManager.d.ts

@@ -0,0 +1,62 @@
+export interface ProgressOptions {
+    title?: string;
+    showETA?: boolean;
+    showSpeed?: boolean;
+    showPercentage?: boolean;
+    width?: number;
+    format?: string;
+}
+export declare class ProgressManager {
+    private static instances;
+    private bar;
+    private startTime;
+    private totalItems;
+    private completed;
+    private id;
+    private title;
+    private constructor();
+    static create(id: string, total: number, options?: ProgressOptions): ProgressManager;
+    static get(id: string): ProgressManager | undefined;
+    update(current: number, detail?: string): void;
+    increment(amount?: number, detail?: string): void;
+    setTotal(total: number): void;
+    stop(showSummary?: boolean): void;
+    fail(error: string): void;
+    getStats(): {
+        completed: number;
+        total: number;
+        percentage: number;
+        duration: number;
+        rate: number;
+        remaining: number;
+        eta: number;
+    };
+    static stopAll(): void;
+}
+export declare class MultiProgressManager {
+    private multiBar;
+    private bars;
+    private startTime;
+    constructor(options?: ProgressOptions);
+    addTask(id: string, total: number, title: string): void;
+    updateTask(id: string, current: number, detail?: string): void;
+    incrementTask(id: string, amount?: number, detail?: string): void;
+    completeTask(id: string): void;
+    failTask(id: string, error: string): void;
+    stop(showSummary?: boolean): void;
+    getTaskStats(id: string): {
+        completed: number;
+        total: number;
+        percentage: number;
+        remaining: number;
+    } | null;
+    getAllStats(): Map<string, any>;
+}
+export declare const ProgressUtils: {
+    withProgress<T>(id: string, total: number, title: string, operation: (progress: ProgressManager) => Promise<T>): Promise<T>;
+    processArrayWithProgress<T, R>(items: T[], processor: (item: T, index: number) => Promise<R>, options?: {
+        title?: string;
+        batchSize?: number;
+        showDetail?: boolean;
+    }): Promise<R[]>;
+};