appwrite-utils-cli 0.10.82 → 0.10.84

package/src/collections/attributes.ts

@@ -5,7 +5,6 @@ import {
   type Attribute,
 } from "appwrite-utils";
 import { nameToIdMapping, enqueueOperation } from "../migrations/queue.js";
-import _ from "lodash";
 import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 import chalk from "chalk";
 
@@ -14,24 +13,24 @@ const attributesSame = (
   configAttribute: Attribute
 ): boolean => {
   const attributesToCheck = [
-    'key',
-    'type',
-    'array',
-    'encrypted',
-    'required',
-    'size',
-    'min',
-    'max',
-    'xdefault',
-    'elements',
-    'relationType',
-    'twoWay',
-    'twoWayKey',
-    'onDelete',
-    'relatedCollection'
+    "key",
+    "type",
+    "array",
+    "encrypted",
+    "required",
+    "size",
+    "min",
+    "max",
+    "xdefault",
+    "elements",
+    "relationType",
+    "twoWay",
+    "twoWayKey",
+    "onDelete",
+    "relatedCollection",
   ];
 
-  return attributesToCheck.every(attr => {
+  return attributesToCheck.every((attr) => {
     // Check if both objects have the attribute
     const dbHasAttr = attr in databaseAttribute;
     const configHasAttr = attr in configAttribute;
@@ -42,7 +41,10 @@ const attributesSame = (
       const configValue = configAttribute[attr as keyof typeof configAttribute];
 
       // Consider undefined and null as equivalent
-      if ((dbValue === undefined || dbValue === null) && (configValue === undefined || configValue === null)) {
+      if (
+        (dbValue === undefined || dbValue === null) &&
+        (configValue === undefined || configValue === null)
+      ) {
         return true;
       }
 
@@ -91,10 +93,18 @@ export const createOrUpdateAttribute = async (
     foundAttribute = undefined;
   }
 
-  if (foundAttribute && attributesSame(foundAttribute, attribute) && updateEnabled) {
+  if (
+    foundAttribute &&
+    attributesSame(foundAttribute, attribute) &&
+    updateEnabled
+  ) {
     // No need to do anything, they are the same
     return;
-  } else if (foundAttribute && !attributesSame(foundAttribute, attribute) && updateEnabled) {
+  } else if (
+    foundAttribute &&
+    !attributesSame(foundAttribute, attribute) &&
+    updateEnabled
+  ) {
     // console.log(
     //   `Updating attribute with same key ${attribute.key} but different values`
     // );
@@ -103,9 +113,15 @@ export const createOrUpdateAttribute = async (
       ...attribute,
     };
     action = "update";
-  } else if (!updateEnabled && foundAttribute && !attributesSame(foundAttribute, attribute)) {
+  } else if (
+    !updateEnabled &&
+    foundAttribute &&
+    !attributesSame(foundAttribute, attribute)
+  ) {
     await db.deleteAttribute(dbId, collection.$id, attribute.key);
-    console.log(`Deleted attribute: ${attribute.key} to recreate it because they diff (update disabled temporarily)`);
+    console.log(
+      `Deleted attribute: ${attribute.key} to recreate it because they diff (update disabled temporarily)`
+    );
     return;
   }
 
@@ -168,7 +184,9 @@ export const createOrUpdateAttribute = async (
               finalAttribute.key,
               finalAttribute.size,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null,
               finalAttribute.array || false,
               finalAttribute.encrypted
             )
@@ -181,7 +199,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null
             )
         );
       }
@@ -209,7 +229,9 @@ export const createOrUpdateAttribute = async (
               finalAttribute.required || false,
               finalAttribute.min || -2147483647,
               finalAttribute.max || 2147483647,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null,
               finalAttribute.array || false
             )
         );
@@ -235,7 +257,9 @@ export const createOrUpdateAttribute = async (
               finalAttribute.required || false,
               finalAttribute.min || -2147483647,
               finalAttribute.max || 2147483647,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null
             )
         );
       }
@@ -251,7 +275,9 @@ export const createOrUpdateAttribute = async (
               finalAttribute.required || false,
               finalAttribute.min || -2147483647,
               finalAttribute.max || 2147483647,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null,
               finalAttribute.array || false
             )
         );
@@ -265,7 +291,9 @@ export const createOrUpdateAttribute = async (
               finalAttribute.required || false,
               finalAttribute.min || -2147483647,
               finalAttribute.max || 2147483647,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null
             )
         );
       }
@@ -279,7 +307,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null,
               finalAttribute.array || false
             )
         );
@@ -291,7 +321,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null
             )
         );
       }
@@ -305,7 +337,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null,
               finalAttribute.array || false
             )
         );
@@ -317,7 +351,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null
             )
         );
       }
@@ -331,7 +367,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null,
               finalAttribute.array || false
             )
         );
@@ -343,7 +381,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null
             )
         );
       }
@@ -357,7 +397,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null,
               finalAttribute.array || false
             )
         );
@@ -369,7 +411,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null
             )
         );
       }
@@ -383,7 +427,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null,
               finalAttribute.array || false
             )
         );
@@ -395,7 +441,9 @@ export const createOrUpdateAttribute = async (
               collection.$id,
               finalAttribute.key,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null
             )
         );
       }
@@ -410,7 +458,9 @@ export const createOrUpdateAttribute = async (
               finalAttribute.key,
               finalAttribute.elements,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null,
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null,
               finalAttribute.array || false
             )
         );
@@ -423,7 +473,9 @@ export const createOrUpdateAttribute = async (
               finalAttribute.key,
               finalAttribute.elements,
               finalAttribute.required || false,
-              finalAttribute.xdefault !== undefined && !finalAttribute.required ? finalAttribute.xdefault : null
+              finalAttribute.xdefault !== undefined && !finalAttribute.required
+                ? finalAttribute.xdefault
+                : null
             )
         );
       }
@@ -468,26 +520,47 @@ export const createUpdateCollectionAttributes = async (
   attributes: Attribute[]
 ): Promise<void> => {
   console.log(
-    chalk.green(`Creating/Updating attributes for collection: ${collection.name}`)
+    chalk.green(
+      `Creating/Updating attributes for collection: ${collection.name}`
+    )
   );
 
-  // @ts-expect-error
-  const existingAttributes: Attribute[] = collection.attributes.map((attr) => parseAttribute(attr)) || [];
+  const existingAttributes: Attribute[] =
+    // @ts-expect-error
+    collection.attributes.map((attr) => parseAttribute(attr)) || [];
 
-  const attributesToRemove = existingAttributes.filter((attr) => !attributes.some((a) => a.key === attr.key));
-  const indexesToRemove = collection.indexes.filter((index) => attributesToRemove.some((attr) => index.attributes.includes(attr.key)));
+  const attributesToRemove = existingAttributes.filter(
+    (attr) => !attributes.some((a) => a.key === attr.key)
+  );
+  const indexesToRemove = collection.indexes.filter((index) =>
+    attributesToRemove.some((attr) => index.attributes.includes(attr.key))
+  );
 
   if (attributesToRemove.length > 0) {
     if (indexesToRemove.length > 0) {
-      console.log(chalk.red(`Removing indexes as they rely on an attribute that is being removed: ${indexesToRemove.map((index) => index.key).join(", ")}`));
+      console.log(
+        chalk.red(
+          `Removing indexes as they rely on an attribute that is being removed: ${indexesToRemove
+            .map((index) => index.key)
+            .join(", ")}`
+        )
+      );
       for (const index of indexesToRemove) {
-        await tryAwaitWithRetry(async () => await db.deleteIndex(dbId, collection.$id, index.key));
+        await tryAwaitWithRetry(
+          async () => await db.deleteIndex(dbId, collection.$id, index.key)
+        );
         await delay(100);
       }
     }
     for (const attr of attributesToRemove) {
-      console.log(chalk.red(`Removing attribute: ${attr.key} as it is no longer in the collection`));
-      await tryAwaitWithRetry(async () => await db.deleteAttribute(dbId, collection.$id, attr.key));
+      console.log(
+        chalk.red(
+          `Removing attribute: ${attr.key} as it is no longer in the collection`
+        )
+      );
+      await tryAwaitWithRetry(
+        async () => await db.deleteAttribute(dbId, collection.$id, attr.key)
+      );
       await delay(50);
     }
   }
@@ -496,7 +569,10 @@ export const createUpdateCollectionAttributes = async (
   for (let i = 0; i < attributes.length; i += batchSize) {
     const batch = attributes.slice(i, i + batchSize);
     const attributePromises = batch.map((attribute) =>
-      tryAwaitWithRetry(async () => await createOrUpdateAttribute(db, dbId, collection, attribute))
+      tryAwaitWithRetry(
+        async () =>
+          await createOrUpdateAttribute(db, dbId, collection, attribute)
+      )
     );
 
     const results = await Promise.allSettled(attributePromises);

package/src/collections/methods.ts

@@ -157,7 +157,7 @@ async function wipeDocumentsFromCollection(
       const docsResponse = await database.listDocuments(
         databaseId,
         collectionId,
-        [Query.limit(1000)]
+        [Query.limit(1000), ...(cursor ? [Query.cursorAfter(cursor)] : [])]
       );
       documents.push(...docsResponse.documents);
       totalDocuments = documents.length;
@@ -391,12 +391,18 @@ export const createOrUpdateCollections = async (
     // Add delay after creating attributes
     await delay(250);
 
+    const indexesToUse =
+      indexes.length > 0
+        ? indexes
+        : config.collections?.find((c) => c.$id === collectionToUse!.$id)
+            ?.indexes ?? [];
+
     console.log("Creating Indexes");
     await createOrUpdateIndexes(
       databaseId,
       database,
       collectionToUse!.$id,
-      (indexes ?? []) as Indexes
+      indexesToUse as Indexes
     );
 
     // Add delay after creating indexes
@@ -455,3 +461,217 @@ export const fetchAllCollections = async (
   console.log(`Fetched a total of ${collections.length} collections.`);
   return collections;
 };
+
+/**
+ * Transfers all documents from one collection to another in a different database
+ * within the same Appwrite Project
+ */
+export const transferDocumentsBetweenDbsLocalToLocal = async (
+  db: Databases,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  let fromCollDocs = await tryAwaitWithRetry(async () =>
+    db.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
+  );
+  let totalDocumentsTransferred = 0;
+
+  if (fromCollDocs.documents.length === 0) {
+    console.log(`No documents found in collection ${fromCollId}`);
+    return;
+  } else if (fromCollDocs.documents.length < 50) {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: Partial<typeof doc> = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(
+        async () =>
+          await db.createDocument(
+            toDbId,
+            toCollId,
+            doc.$id,
+            toCreateObject,
+            doc.$permissions
+          )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+  } else {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: Partial<typeof doc> = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(async () =>
+        db.createDocument(
+          toDbId,
+          toCollId,
+          doc.$id,
+          toCreateObject,
+          doc.$permissions
+        )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+    while (fromCollDocs.documents.length === 50) {
+      fromCollDocs = await tryAwaitWithRetry(
+        async () =>
+          await db.listDocuments(fromDbId, fromCollId, [
+            Query.limit(50),
+            Query.cursorAfter(
+              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
+            ),
+          ])
+      );
+      const batchedPromises = fromCollDocs.documents.map((doc) => {
+        const toCreateObject: Partial<typeof doc> = {
+          ...doc,
+        };
+        delete toCreateObject.$databaseId;
+        delete toCreateObject.$collectionId;
+        delete toCreateObject.$createdAt;
+        delete toCreateObject.$updatedAt;
+        delete toCreateObject.$id;
+        delete toCreateObject.$permissions;
+        return tryAwaitWithRetry(
+          async () =>
+            await db.createDocument(
+              toDbId,
+              toCollId,
+              doc.$id,
+              toCreateObject,
+              doc.$permissions
+            )
+        );
+      });
+      await Promise.all(batchedPromises);
+      totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+  }
+
+  console.log(
+    `Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`
+  );
+};
+
+export const transferDocumentsBetweenDbsLocalToRemote = async (
+  localDb: Databases,
+  endpoint: string,
+  projectId: string,
+  apiKey: string,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  const client = new Client()
+    .setEndpoint(endpoint)
+    .setProject(projectId)
+    .setKey(apiKey);
+  let totalDocumentsTransferred = 0;
+  const remoteDb = new Databases(client);
+  let fromCollDocs = await tryAwaitWithRetry(async () =>
+    localDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
+  );
+
+  if (fromCollDocs.documents.length === 0) {
+    console.log(`No documents found in collection ${fromCollId}`);
+    return;
+  } else if (fromCollDocs.documents.length < 50) {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: Partial<typeof doc> = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(async () =>
+        remoteDb.createDocument(
+          toDbId,
+          toCollId,
+          doc.$id,
+          toCreateObject,
+          doc.$permissions
+        )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+  } else {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: Partial<typeof doc> = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(async () =>
+        remoteDb.createDocument(
+          toDbId,
+          toCollId,
+          doc.$id,
+          toCreateObject,
+          doc.$permissions
+        )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+    while (fromCollDocs.documents.length === 50) {
+      fromCollDocs = await tryAwaitWithRetry(async () =>
+        localDb.listDocuments(fromDbId, fromCollId, [
+          Query.limit(50),
+          Query.cursorAfter(
+            fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
+          ),
+        ])
+      );
+      const batchedPromises = fromCollDocs.documents.map((doc) => {
+        const toCreateObject: Partial<typeof doc> = {
+          ...doc,
+        };
+        delete toCreateObject.$databaseId;
+        delete toCreateObject.$collectionId;
+        delete toCreateObject.$createdAt;
+        delete toCreateObject.$updatedAt;
+        delete toCreateObject.$id;
+        delete toCreateObject.$permissions;
+        return tryAwaitWithRetry(async () =>
+          remoteDb.createDocument(
+            toDbId,
+            toCollId,
+            doc.$id,
+            toCreateObject,
+            doc.$permissions
+          )
+        );
+      });
+      await Promise.all(batchedPromises);
+      totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+  }
+  console.log(
+    `Total documents transferred from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}: ${totalDocumentsTransferred}`
+  );
+};

package/src/migrations/appwriteToX.ts

@@ -8,7 +8,7 @@ import {
   type Models,
   type Permission,
 } from "node-appwrite";
-import { fetchAllCollections } from "./collections.js";
+import { fetchAllCollections } from "../collections/methods.js";
 import { fetchAllDatabases } from "./databases.js";
 import {
   CollectionSchema,
@@ -138,7 +138,7 @@ export class AppwriteToX {
           .map((attr: any) => {
             return parseAttribute(attr);
           })
-          .filter((attribute) =>
+          .filter((attribute: Attribute) =>
             attribute.type === "relationship"
               ? attribute.side !== "child"
               : true
@@ -169,10 +169,10 @@ export class AppwriteToX {
           }
         }
         this.collToAttributeMap.set(collection.name, collAttributes);
-        const finalIndexes = collection.indexes.map((index) => {
+        const finalIndexes = collection.indexes.map((index: Models.Index) => {
           return {
             ...index,
-            orders: index.orders?.filter((order) => {
+            orders: index.orders?.filter((order: string) => {
               return order !== null && order;
             }),
           };

package/src/migrations/attributes.ts

@@ -5,7 +5,6 @@ import {
   type Attribute,
 } from "appwrite-utils";
 import { nameToIdMapping, enqueueOperation } from "./queue.js";
-import _ from "lodash";
 import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 const attributesSame = (

package/src/migrations/converters.ts

@@ -1,7 +1,5 @@
-import _ from "lodash";
 import { converterFunctions, type AttributeMappings } from "appwrite-utils";
-
-const { cloneDeep, isObject } = _;
+import { cloneDeep, isPlainObject } from "es-toolkit";
 
 /**
  * Deeply converts all properties of an object (or array) to strings.
@@ -11,7 +9,7 @@ const { cloneDeep, isObject } = _;
 export const deepAnyToString = (data: any): any => {
   if (Array.isArray(data)) {
     return data.map((item) => deepAnyToString(item));
-  } else if (isObject(data)) {
+  } else if (isPlainObject(data)) {
     return Object.keys(data).reduce((acc, key) => {
       acc[key] = deepAnyToString(data[key as keyof typeof data]);
       return acc;
@@ -34,7 +32,7 @@ export const deepConvert = <T>(
 ): any => {
   if (Array.isArray(data)) {
     return data.map((item) => deepConvert(item, convertFn));
-  } else if (isObject(data)) {
+  } else if (isPlainObject(data)) {
     return Object.keys(data).reduce((acc: Record<string, T>, key: string) => {
       acc[key] = deepConvert(data[key as keyof typeof data], convertFn);
       return acc;