appwrite-utils-cli 0.0.1

package/dist/migrations/migrationHelper.js

@@ -0,0 +1,66 @@
+import { ID, Query } from "node-appwrite";
+import { OperationSchema } from "./backup.js";
+export const findOrCreateOperation = async (database, collectionId, operationType) => {
+    // Here you would query your database for an existing operation
+    // If it doesn't exist, create a new one
+    // This is a simplified example
+    const operations = await database.listDocuments("migrations", "currentOperations", [
+        Query.equal("collectionId", collectionId),
+        Query.equal("operationType", operationType),
+        Query.equal("status", "in_progress"),
+    ]);
+    if (operations.documents.length > 0) {
+        return OperationSchema.parse(operations.documents[0]); // Assuming the first document is the operation we want
+    }
+    else {
+        // Create a new operation document
+        const op = await database.createDocument("migrations", "currentOperations", ID.unique(), {
+            operationType,
+            collectionId,
+            status: "in_progress",
+            batches: [],
+            progress: 0,
+            total: 0,
+            error: "",
+        });
+        return OperationSchema.parse(op);
+    }
+};
+export const updateOperation = async (database, operationId, updateFields) => {
+    await database.updateDocument("migrations", "currentOperations", operationId, updateFields);
+};
+// Actual max 1073741824
+export const maxDataLength = 1073741820;
+export const maxBatchItems = 100;
+export const splitIntoBatches = (data) => {
+    let batches = [];
+    let currentBatch = [];
+    let currentBatchLength = 0;
+    let currentBatchItemCount = 0;
+    data.forEach((item, index) => {
+        const itemLength = JSON.stringify(item).length;
+        if (itemLength > maxDataLength) {
+            console.log(item, `Large item found at index ${index} with length ${itemLength}:`);
+        }
+        // Check if adding the current item would exceed the max length or max items per batch
+        if (currentBatchLength + itemLength >= maxDataLength ||
+            currentBatchItemCount >= maxBatchItems) {
+            // If so, start a new batch
+            batches.push(currentBatch);
+            currentBatch = [item];
+            currentBatchLength = itemLength;
+            currentBatchItemCount = 1; // Reset item count for the new batch
+        }
+        else {
+            // Otherwise, add the item to the current batch
+            currentBatch.push(item);
+            currentBatchLength += itemLength;
+            currentBatchItemCount++;
+        }
+    });
+    // Don't forget to add the last batch if it's not empty
+    if (currentBatch.length > 0) {
+        batches.push(currentBatch);
+    }
+    return batches;
+};

package/dist/migrations/queue.d.ts

@@ -0,0 +1,13 @@
+import { type Databases, type Models } from "node-appwrite";
+import type { Attribute } from "./schema.js";
+export interface QueuedOperation {
+    type: "attribute";
+    collectionId?: string;
+    attribute?: Attribute;
+    collection?: Models.Collection;
+    dependencies?: string[];
+}
+export declare const queuedOperations: QueuedOperation[];
+export declare const nameToIdMapping: Map<string, string>;
+export declare const enqueueOperation: (operation: QueuedOperation) => void;
+export declare const processQueue: (db: Databases, dbId: string) => Promise<void>;

package/dist/migrations/queue.js

@@ -0,0 +1,79 @@
+import { Query } from "node-appwrite";
+import { createOrUpdateAttribute } from "./attributes.js";
+import _ from "lodash";
+import { fetchAndCacheCollectionByName } from "./collections.js";
+export const queuedOperations = [];
+export const nameToIdMapping = new Map();
+export const enqueueOperation = (operation) => {
+    queuedOperations.push(operation);
+};
+export const processQueue = async (db, dbId) => {
+    console.log("---------------------------------");
+    console.log(`Starting Queue processing of ${dbId}`);
+    console.log("---------------------------------");
+    let progress = true;
+    while (progress) {
+        progress = false;
+        console.log("Processing queued operations:");
+        for (let i = 0; i < queuedOperations.length; i++) {
+            const operation = queuedOperations[i];
+            let collectionFound;
+            // Handle relationship attribute operations
+            if (operation.attribute?.type === "relationship") {
+                // Attempt to resolve the collection directly if collectionId is specified
+                if (operation.collectionId) {
+                    console.log(`\tFetching collection by ID: ${operation.collectionId}`);
+                    try {
+                        collectionFound = await db.getCollection(dbId, operation.collectionId);
+                    }
+                    catch (e) {
+                        console.log(`\tCollection not found by ID: ${operation.collectionId}`);
+                    }
+                }
+                // Attempt to resolve related collection if specified and not already found
+                if (!collectionFound && operation.attribute?.relatedCollection) {
+                    collectionFound = await fetchAndCacheCollectionByName(db, dbId, operation.attribute.relatedCollection);
+                }
+                // Handle dependencies if collection still not found
+                if (!collectionFound) {
+                    for (const dep of operation.dependencies || []) {
+                        collectionFound = await fetchAndCacheCollectionByName(db, dbId, dep);
+                        if (collectionFound)
+                            break; // Break early if collection is found
+                    }
+                }
+            }
+            else if (operation.collectionId) {
+                // Handle non-relationship operations with a specified collectionId
+                console.log(`\tFetching collection by ID: ${operation.collectionId}`);
+                try {
+                    collectionFound = await db.getCollection(dbId, operation.collectionId);
+                }
+                catch (e) {
+                    console.log(`\tCollection not found by ID: ${operation.collectionId}`);
+                }
+            }
+            // Process the operation if the collection is found
+            if (collectionFound && operation.attribute) {
+                console.log(`\tProcessing attribute: ${operation.attribute.key} for collection ID: ${collectionFound.$id}`);
+                await createOrUpdateAttribute(db, dbId, collectionFound, operation.attribute);
+                queuedOperations.splice(i, 1);
+                i--; // Adjust index since we're modifying the array
+                progress = true;
+            }
+            else {
+                console.error(`\tCollection not found for operation, removing from queue: ${JSON.stringify(operation)}`);
+                queuedOperations.splice(i, 1);
+                i--; // Adjust index since we're modifying the array
+            }
+        }
+        console.log(`\tFinished processing queued operations`);
+    }
+    if (queuedOperations.length > 0) {
+        console.error("Unresolved operations remain due to unmet dependencies.");
+        console.log(queuedOperations);
+    }
+    console.log("---------------------------------");
+    console.log(`Queue processing complete for ${dbId}`);
+    console.log("---------------------------------");
+};

package/dist/migrations/relationships.d.ts

@@ -0,0 +1,90 @@
+import { Databases } from "node-appwrite";
+import type { AppwriteConfig } from "./schema.js";
+export declare const findCollectionsWithRelationships: (config: AppwriteConfig) => Map<string, ({
+    key: string;
+    type: "string";
+    error: string;
+    required: boolean;
+    array: boolean;
+    size: number;
+    xdefault?: string | null | undefined;
+    encrypted?: boolean | undefined;
+} | {
+    key: string;
+    type: "integer";
+    error: string;
+    required: boolean;
+    array: boolean;
+    min?: number | undefined;
+    max?: number | undefined;
+    xdefault?: number | null | undefined;
+} | {
+    key: string;
+    type: "float";
+    error: string;
+    required: boolean;
+    array: boolean;
+    min?: number | undefined;
+    max?: number | undefined;
+    xdefault?: number | null | undefined;
+} | {
+    key: string;
+    type: "boolean";
+    error: string;
+    required: boolean;
+    array: boolean;
+    xdefault?: boolean | null | undefined;
+} | {
+    key: string;
+    type: "datetime";
+    error: string;
+    required: boolean;
+    array: boolean;
+    xdefault?: string | null | undefined;
+} | {
+    key: string;
+    type: "email";
+    error: string;
+    required: boolean;
+    array: boolean;
+    xdefault?: string | null | undefined;
+} | {
+    key: string;
+    type: "ip";
+    error: string;
+    required: boolean;
+    array: boolean;
+    xdefault?: string | null | undefined;
+} | {
+    key: string;
+    type: "url";
+    error: string;
+    required: boolean;
+    array: boolean;
+    xdefault?: string | null | undefined;
+} | {
+    key: string;
+    type: "enum";
+    error: string;
+    required: boolean;
+    array: boolean;
+    elements: string[];
+    xdefault?: string | null | undefined;
+} | {
+    key: string;
+    type: "relationship";
+    error: string;
+    required: boolean;
+    relatedCollection: string;
+    relationType: "oneToMany" | "manyToOne" | "oneToOne" | "manyToMany";
+    twoWay: boolean;
+    twoWayKey: string;
+    onDelete: "setNull" | "cascade" | "restrict";
+    side: "parent" | "child";
+    array?: boolean | undefined;
+    importMapping?: {
+        originalIdField: string;
+        targetField?: string | undefined;
+    } | undefined;
+})[]>;
+export declare function resolveAndUpdateRelationships(dbId: string, database: Databases, config: AppwriteConfig): Promise<void>;

package/dist/migrations/relationships.js

@@ -0,0 +1,209 @@
+import { Databases, Query } from "node-appwrite";
+import { fetchAllCollections } from "./collections.js";
+export const findCollectionsWithRelationships = (config) => {
+    const toReturn = new Map();
+    // Map of collection name to array of attributes so we can update the relationships
+    for (const collection of config.collections) {
+        if (collection.attributes) {
+            for (const attribute of collection.attributes) {
+                if (attribute.type === "relationship") {
+                    if (!toReturn.has(collection.name)) {
+                        toReturn.set(collection.name, []);
+                    }
+                    toReturn.get(collection.name)?.push(attribute);
+                    if (!toReturn.has(attribute.relatedCollection)) {
+                        toReturn.set(attribute.relatedCollection, []);
+                    }
+                    toReturn.get(attribute.relatedCollection)?.push(attribute);
+                }
+            }
+        }
+    }
+    return toReturn;
+};
+async function fetchAllDocuments(dbId, database, collectionId) {
+    let allDocuments = [];
+    let after; // This will be used for pagination
+    while (true) {
+        const response = await database.listDocuments(dbId, collectionId, [
+            Query.limit(100), // Adjust based on the maximum limit your database allows
+            ...(after ? [Query.cursorAfter(after)] : []),
+        ]);
+        allDocuments = allDocuments.concat(response.documents);
+        if (response.documents.length === 0 || response.total === 0) {
+            break; // Exit the loop if there are no more documents to fetch
+        }
+        after = response.documents[response.documents.length - 1].$id; // Prepare for the next page
+    }
+    return allDocuments;
+}
+export async function resolveAndUpdateRelationships(dbId, database, config) {
+    const collections = await fetchAllCollections(dbId, database);
+    const collectionsWithRelationships = findCollectionsWithRelationships(config);
+    // Process each collection sequentially
+    for (const collection of collections) {
+        console.log(`Processing collection: ${collection.name} (${collection.$id})`);
+        const relAttributeMap = collectionsWithRelationships.get(collection.name); // Get the relationship attributes for the collections
+        if (!relAttributeMap) {
+            console.log(`No mapping found for collection: ${collection.name}, skipping...`);
+            continue;
+        }
+        await processCollection(dbId, database, collection, relAttributeMap);
+    }
+    console.log(`Completed relationship resolution and update for database ID: ${dbId}`);
+}
+async function processCollection(dbId, database, collection, relAttributeMap) {
+    const allDocuments = await fetchAllDocuments(dbId, database, collection.$id);
+    console.log(`Fetched ${allDocuments.length} documents from collection: ${collection.name}`);
+    const batchSize = 10; // Process documents in batches of 10
+    for (let i = 0; i < allDocuments.length; i += batchSize) {
+        const batch = allDocuments.slice(i, i + batchSize);
+        const updates = await prepareDocumentUpdates(database, dbId, collection.name, batch, relAttributeMap);
+        // Execute updates for the current batch
+        await executeUpdatesInBatches(dbId, database, updates);
+    }
+}
+async function findDocumentsByOriginalId(database, dbId, relatedCollectionId, targetKey, originalId) {
+    const collection = await database.listCollections(dbId, [
+        Query.equal("$id", relatedCollectionId),
+    ]);
+    if (collection.total === 0) {
+        console.log(`Collection ${relatedCollectionId} doesn't exist, skipping...`);
+        return undefined;
+    }
+    const targetAttr = collection.collections[0].attributes.find(
+    // @ts-ignore
+    (attr) => attr.key === targetKey);
+    if (!targetAttr) {
+        console.log(`Attribute ${targetKey} not found in collection ${relatedCollectionId}, skipping...`);
+        return undefined;
+    }
+    let queries = [];
+    if (targetAttr.array) {
+        // @ts-ignore
+        queries.push(Query.contains(targetKey, originalId));
+    }
+    else {
+        queries.push(Query.equal(targetKey, originalId));
+    }
+    const response = await database.listDocuments(dbId, relatedCollectionId, [
+        ...queries,
+        Query.limit(500), // Adjust the limit based on your needs or implement pagination
+    ]);
+    if (response.total > 0) {
+        return undefined;
+    }
+    if (response.documents.length > 0) {
+        return response.documents;
+    }
+    else {
+        return undefined;
+    }
+}
+async function prepareDocumentUpdates(database, dbId, collectionName, documents, relationships) {
+    console.log(`Preparing updates for collection: ${collectionName}`);
+    const updates = [];
+    const thisCollection = (await database.listCollections(dbId, [Query.equal("name", collectionName)])).collections[0];
+    const thisCollectionId = thisCollection?.$id;
+    if (!thisCollectionId) {
+        console.log(`No collection found with name: ${collectionName}`);
+        return [];
+    }
+    // Function to process a batch of documents
+    const processDocumentBatch = async (docBatch) => {
+        for (const doc of docBatch) {
+            let updatePayload = {};
+            for (const rel of relationships) {
+                // Check if the relationship has importMapping defined
+                if (!rel.importMapping) {
+                    continue;
+                }
+                // Skip if not dealing with the parent side of a two-way relationship
+                if (rel.twoWay && rel.side !== "parent") {
+                    console.log("Not processing child side of two-way relationship, skipping...");
+                    continue;
+                }
+                const isSingleReference = rel.relationType === "oneToOne" || rel.relationType === "manyToOne";
+                const originalIdField = rel.importMapping.originalIdField;
+                const targetField = rel.importMapping.targetField || originalIdField; // Use originalIdField if targetField is not specified
+                const originalId = doc[originalIdField];
+                if (!originalId) {
+                    continue; // Skip if the document doesn't have the original ID field
+                }
+                const collection = await database.listCollections(dbId, [
+                    Query.equal("name", rel.relatedCollection),
+                ]);
+                if (collection.total === 0) {
+                    console.log(`Collection ${rel.relatedCollection} doesn't exist, skipping...`);
+                    continue; // Skip if the related collection doesn't exist
+                }
+                const relatedCollectionId = collection.collections[0].$id;
+                // Find documents in the related collection that match the original ID
+                const foundDocuments = await findDocumentsByOriginalId(database, dbId, relatedCollectionId, targetField, originalId);
+                if (foundDocuments && foundDocuments.length > 0) {
+                    const relationshipKey = rel.key;
+                    const existingRefs = doc[relationshipKey] || [];
+                    let existingRefIds = [];
+                    if (Array.isArray(existingRefs)) {
+                        // @ts-ignore
+                        existingRefIds = existingRefs.map((ref) => ref.$id);
+                    }
+                    else if (existingRefs) {
+                        // @ts-ignore
+                        existingRefIds = [existingRefs.$id];
+                    }
+                    const newRefs = foundDocuments.map((fd) => fd.$id);
+                    const allRefs = [...new Set([...existingRefIds, ...newRefs])]; // Combine and remove duplicates
+                    // Update logic based on the relationship cardinality
+                    updatePayload[relationshipKey] = isSingleReference
+                        ? newRefs[0] || existingRefIds[0]
+                        : allRefs;
+                }
+            }
+            if (Object.keys(updatePayload).length > 0) {
+                updates.push({
+                    collectionId: thisCollectionId,
+                    documentId: doc.$id,
+                    updatePayload: updatePayload,
+                });
+            }
+        }
+    };
+    // Process documents in batches
+    await processInBatches(documents, 25, processDocumentBatch);
+    return updates;
+}
+async function processInBatches(items, batchSize, processFunction) {
+    const maxParallelBatches = 25; // Adjust this value to control the number of parallel batches
+    let currentIndex = 0;
+    let activeBatchPromises = [];
+    while (currentIndex < items.length) {
+        // While there's still data to process and we haven't reached our parallel limit
+        while (currentIndex < items.length &&
+            activeBatchPromises.length < maxParallelBatches) {
+            const batch = items.slice(currentIndex, currentIndex + batchSize);
+            currentIndex += batchSize;
+            // Add new batch processing promise to the array
+            activeBatchPromises.push(processFunction(batch));
+        }
+        // Wait for one of the batch processes to complete
+        await Promise.race(activeBatchPromises).then(() => {
+            // Remove the resolved promise from the activeBatchPromises array
+            activeBatchPromises = activeBatchPromises.filter((p) => p !== Promise.race(activeBatchPromises));
+        });
+    }
+    // After processing all batches, ensure all active promises are resolved
+    await Promise.all(activeBatchPromises);
+}
+async function executeUpdatesInBatches(dbId, database, updates) {
+    const batchSize = 25; // Adjust based on your rate limit and performance testing
+    for (let i = 0; i < updates.length; i += batchSize) {
+        const batch = updates.slice(i, i + batchSize);
+        await Promise.all(batch.map((update) => database
+            .updateDocument(dbId, update.collectionId, update.documentId, update.updatePayload)
+            .catch((error) => {
+            console.error("Error updating document: ", error);
+            console.error("Document ID: ", update.documentId, "Collection ID: ", update.collectionId, "Document update payload: ", JSON.stringify(update.updatePayload, undefined, 4));
+        })));
+    }
+}