appwrite-utils-cli 0.0.285 → 0.9.0

package/dist/migrations/collections.js

@@ -1,9 +1,10 @@
-import { Databases, ID, Permission, Query } from "node-appwrite";
+import { Client, Databases, ID, Permission, Query, } from "node-appwrite";
 import { nameToIdMapping, processQueue } from "./queue.js";
 import { createUpdateCollectionAttributes } from "./attributes.js";
 import { createOrUpdateIndexes } from "./indexes.js";
 import _ from "lodash";
 import { SchemaGenerator } from "./schemaStrings.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export const documentExists = async (db, dbId, targetCollectionId, toCreateObject) => {
     // Had to do this because kept running into issues with type checking arrays so, sorry 40ms
     const collection = await db.getCollection(dbId, targetCollectionId);
@@ -48,9 +49,7 @@ export const documentExists = async (db, dbId, targetCollectionId, toCreateObjec
 export const checkForCollection = async (db, dbId, collection) => {
     try {
         console.log(`Checking for collection with name: ${collection.name}`);
-        const response = await db.listCollections(dbId, [
-            Query.equal("name", collection.name),
-        ]);
+        const response = await tryAwaitWithRetry(async () => await db.listCollections(dbId, [Query.equal("name", collection.name)]));
         if (response.collections.length > 0) {
             console.log(`Collection found: ${response.collections[0].$id}`);
             return { ...collection, ...response.collections[0] };
@@ -70,13 +69,11 @@ export const fetchAndCacheCollectionByName = async (db, dbId, collectionName) =>
     if (nameToIdMapping.has(collectionName)) {
         const collectionId = nameToIdMapping.get(collectionName);
         console.log(`\tCollection found in cache: ${collectionId}`);
-        return await db.getCollection(dbId, collectionId);
+        return await tryAwaitWithRetry(async () => await db.getCollection(dbId, collectionId));
     }
     else {
         console.log(`\tFetching collection by name: ${collectionName}`);
-        const collectionsPulled = await db.listCollections(dbId, [
-            Query.equal("name", collectionName),
-        ]);
+        const collectionsPulled = await tryAwaitWithRetry(async () => await db.listCollections(dbId, [Query.equal("name", collectionName)]));
         if (collectionsPulled.total > 0) {
             const collection = collectionsPulled.collections[0];
             console.log(`\tCollection found: ${collection.$id}`);
@@ -91,7 +88,7 @@ export const fetchAndCacheCollectionByName = async (db, dbId, collectionName) =>
 };
 export const wipeDatabase = async (database, databaseId) => {
     console.log(`Wiping database: ${databaseId}`);
-    const { collections: existingCollections } = await database.listCollections(databaseId);
+    const existingCollections = await fetchAllCollections(databaseId, database);
     let collectionsDeleted = [];
     for (const { $id: collectionId, name: name } of existingCollections) {
         console.log(`Deleting collection: ${collectionId}`);
@@ -99,7 +96,7 @@ export const wipeDatabase = async (database, databaseId) => {
             collectionId: collectionId,
             collectionName: name,
         });
-        await database.deleteCollection(databaseId, collectionId);
+        tryAwaitWithRetry(async () => await database.deleteCollection(databaseId, collectionId)); // Try to delete the collection and ignore errors if it doesn't exist or if it's already being deleted
     }
     return collectionsDeleted;
 };
@@ -141,21 +138,21 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
             }
         }
         // Check if the collection already exists by name
-        let collectionsFound = await database.listCollections(databaseId, [
+        let collectionsFound = await tryAwaitWithRetry(async () => await database.listCollections(databaseId, [
             Query.equal("name", collection.name),
-        ]);
+        ]));
         let collectionToUse = collectionsFound.total > 0 ? collectionsFound.collections[0] : null;
         // Determine the correct ID for the collection
         let collectionId;
         if (!collectionToUse) {
             console.log(`Creating collection: ${collection.name}`);
-            const foundColl = deletedCollections?.find((coll) => coll.collectionName.toLowerCase().trim().replace(" ", "") ===
+            let foundColl = deletedCollections?.find((coll) => coll.collectionName.toLowerCase().trim().replace(" ", "") ===
                 collection.name.toLowerCase().trim().replace(" ", ""));
-            if (foundColl && !usedIds.has(foundColl.collectionId)) {
-                collectionId = foundColl.collectionId; // Use ID from deleted collection if not already used
+            if (collection.$id) {
+                collectionId = collection.$id; // Always use the provided $id if present
             }
-            else if (collection.$id && !usedIds.has(collection.$id)) {
-                collectionId = collection.$id; // Use the provided $id if not already used
+            else if (foundColl && !usedIds.has(foundColl.collectionId)) {
+                collectionId = foundColl.collectionId; // Use ID from deleted collection if not already used
             }
             else {
                 collectionId = ID.unique(); // Generate a new unique ID
@@ -163,7 +160,7 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
             usedIds.add(collectionId); // Mark this ID as used
             // Create the collection with the determined ID
             try {
-                collectionToUse = await database.createCollection(databaseId, collectionId, collection.name, permissions, collection.documentSecurity, collection.enabled);
+                collectionToUse = await tryAwaitWithRetry(async () => await database.createCollection(databaseId, collectionId, collection.name, permissions, collection.documentSecurity ?? false, collection.enabled ?? true));
                 collection.$id = collectionToUse.$id;
                 nameToIdMapping.set(collection.name, collectionToUse.$id);
             }
@@ -173,7 +170,8 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
             }
         }
         else {
-            console.log(`Collection ${collection.name} exists, using it`);
+            console.log(`Collection ${collection.name} exists, updating it`);
+            await tryAwaitWithRetry(async () => await database.updateCollection(databaseId, collectionToUse.$id, collection.name, permissions, collection.documentSecurity ?? false, collection.enabled ?? true));
         }
         // Update attributes and indexes for the collection
         console.log("Creating Attributes");
@@ -205,7 +203,7 @@ export const fetchAllCollections = async (dbId, database) => {
         if (lastCollectionId) {
             queries.push(Query.cursorAfter(lastCollectionId));
         }
-        const response = await database.listCollections(dbId, queries);
+        const response = await tryAwaitWithRetry(async () => await database.listCollections(dbId, queries));
         collections = collections.concat(response.collections);
         moreCollections = response.collections.length === 500;
         if (moreCollections) {
@@ -216,3 +214,134 @@ export const fetchAllCollections = async (dbId, database) => {
     console.log(`Fetched a total of ${collections.length} collections.`);
     return collections;
 };
+/**
+ * Transfers all documents from one collection to another in a different database
+ * within the same Appwrite Project
+ */
+export const transferDocumentsBetweenDbsLocalToLocal = async (db, fromDbId, toDbId, fromCollId, toCollId) => {
+    let fromCollDocs = await tryAwaitWithRetry(async () => db.listDocuments(fromDbId, fromCollId, [Query.limit(50)]));
+    let totalDocumentsTransferred = 0;
+    if (fromCollDocs.documents.length === 0) {
+        console.log(`No documents found in collection ${fromCollId}`);
+        return;
+    }
+    else if (fromCollDocs.documents.length < 50) {
+        const batchedPromises = fromCollDocs.documents.map((doc) => {
+            const toCreateObject = {
+                ...doc,
+            };
+            delete toCreateObject.$databaseId;
+            delete toCreateObject.$collectionId;
+            delete toCreateObject.$createdAt;
+            delete toCreateObject.$updatedAt;
+            delete toCreateObject.$id;
+            delete toCreateObject.$permissions;
+            return tryAwaitWithRetry(async () => await db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+        });
+        await Promise.all(batchedPromises);
+        totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+    else {
+        const batchedPromises = fromCollDocs.documents.map((doc) => {
+            const toCreateObject = {
+                ...doc,
+            };
+            delete toCreateObject.$databaseId;
+            delete toCreateObject.$collectionId;
+            delete toCreateObject.$createdAt;
+            delete toCreateObject.$updatedAt;
+            delete toCreateObject.$id;
+            delete toCreateObject.$permissions;
+            return tryAwaitWithRetry(async () => db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+        });
+        await Promise.all(batchedPromises);
+        totalDocumentsTransferred += fromCollDocs.documents.length;
+        while (fromCollDocs.documents.length === 50) {
+            fromCollDocs = await tryAwaitWithRetry(async () => await db.listDocuments(fromDbId, fromCollId, [
+                Query.limit(50),
+                Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
+            ]));
+            const batchedPromises = fromCollDocs.documents.map((doc) => {
+                const toCreateObject = {
+                    ...doc,
+                };
+                delete toCreateObject.$databaseId;
+                delete toCreateObject.$collectionId;
+                delete toCreateObject.$createdAt;
+                delete toCreateObject.$updatedAt;
+                delete toCreateObject.$id;
+                delete toCreateObject.$permissions;
+                return tryAwaitWithRetry(async () => await db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+            });
+            await Promise.all(batchedPromises);
+            totalDocumentsTransferred += fromCollDocs.documents.length;
+        }
+    }
+    console.log(`Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`);
+};
+export const transferDocumentsBetweenDbsLocalToRemote = async (localDb, endpoint, projectId, apiKey, fromDbId, toDbId, fromCollId, toCollId) => {
+    const client = new Client()
+        .setEndpoint(endpoint)
+        .setProject(projectId)
+        .setKey(apiKey);
+    let totalDocumentsTransferred = 0;
+    const remoteDb = new Databases(client);
+    let fromCollDocs = await tryAwaitWithRetry(async () => localDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)]));
+    if (fromCollDocs.documents.length === 0) {
+        console.log(`No documents found in collection ${fromCollId}`);
+        return;
+    }
+    else if (fromCollDocs.documents.length < 50) {
+        const batchedPromises = fromCollDocs.documents.map((doc) => {
+            const toCreateObject = {
+                ...doc,
+            };
+            delete toCreateObject.$databaseId;
+            delete toCreateObject.$collectionId;
+            delete toCreateObject.$createdAt;
+            delete toCreateObject.$updatedAt;
+            delete toCreateObject.$id;
+            delete toCreateObject.$permissions;
+            return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+        });
+        await Promise.all(batchedPromises);
+        totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+    else {
+        const batchedPromises = fromCollDocs.documents.map((doc) => {
+            const toCreateObject = {
+                ...doc,
+            };
+            delete toCreateObject.$databaseId;
+            delete toCreateObject.$collectionId;
+            delete toCreateObject.$createdAt;
+            delete toCreateObject.$updatedAt;
+            delete toCreateObject.$id;
+            delete toCreateObject.$permissions;
+            return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+        });
+        await Promise.all(batchedPromises);
+        totalDocumentsTransferred += fromCollDocs.documents.length;
+        while (fromCollDocs.documents.length === 50) {
+            fromCollDocs = await tryAwaitWithRetry(async () => localDb.listDocuments(fromDbId, fromCollId, [
+                Query.limit(50),
+                Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
+            ]));
+            const batchedPromises = fromCollDocs.documents.map((doc) => {
+                const toCreateObject = {
+                    ...doc,
+                };
+                delete toCreateObject.$databaseId;
+                delete toCreateObject.$collectionId;
+                delete toCreateObject.$createdAt;
+                delete toCreateObject.$updatedAt;
+                delete toCreateObject.$id;
+                delete toCreateObject.$permissions;
+                return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
+            });
+            await Promise.all(batchedPromises);
+            totalDocumentsTransferred += fromCollDocs.documents.length;
+        }
+    }
+    console.log(`Total documents transferred from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}: ${totalDocumentsTransferred}`);
+};

package/dist/migrations/converters.d.ts

@@ -1,3 +1,4 @@
+import { type AttributeMappings } from "appwrite-utils";
 /**
  * Deeply converts all properties of an object (or array) to strings.
  * @param data The input data to convert.
@@ -28,24 +29,7 @@ export declare const convertObjectBySchema: (obj: Record<string, any>, schema: R
  * @param attributeMappings The attributeMappings defining how keys in the object should be converted.
  * @returns The converted object with keys renamed according to attributeMappings.
  */
-export declare const convertObjectByAttributeMappings: (obj: Record<string, any>, attributeMappings: {
-    targetKey: string;
-    oldKey?: string | undefined;
-    oldKeys?: string[] | undefined;
-    fileData?: {
-        path: string;
-        name: string;
-    } | undefined;
-    converters?: string[] | undefined;
-    validationActions?: {
-        params: string[];
-        action: string;
-    }[] | undefined;
-    postImportActions?: {
-        params: (string | Record<string, any>)[];
-        action: string;
-    }[] | undefined;
-}[]) => Record<string, any>;
+export declare const convertObjectByAttributeMappings: (obj: Record<string, any>, attributeMappings: AttributeMappings) => Record<string, any>;
 /**
  * Ensures data conversion without mutating the original input.
  * @param data The data to convert.

package/dist/migrations/converters.js

@@ -89,12 +89,20 @@ export const convertObjectByAttributeMappings = (obj, attributeMappings) => {
         return current;
     };
     for (const mapping of attributeMappings) {
-        if (Array.isArray(mapping.oldKeys)) {
+        if (mapping.valueToSet !== undefined) {
+            result[mapping.targetKey] = mapping.valueToSet;
+        }
+        else if (Array.isArray(mapping.oldKeys)) {
             // Collect and flatten values from multiple oldKeys
             const values = mapping.oldKeys
                 .map((oldKey) => resolveValue(obj, oldKey))
                 .flat(Infinity);
-            result[mapping.targetKey] = values.filter((value) => value !== undefined);
+            if (values.length > 0) {
+                result[mapping.targetKey] = values.filter((value) => value !== undefined);
+            }
+            else {
+                result[mapping.targetKey] = null;
+            }
         }
         else if (mapping.oldKey) {
             // Resolve single oldKey
@@ -104,6 +112,9 @@ export const convertObjectByAttributeMappings = (obj, attributeMappings) => {
                     ? value.flat(Infinity)
                     : value;
             }
+            else {
+                result[mapping.targetKey] = value ? value : null;
+            }
         }
     }
     return result;