appwrite-utils-cli 0.10.81 → 0.10.83

package/src/collections/methods.ts

@@ -10,8 +10,15 @@ import type { AppwriteConfig, CollectionCreate, Indexes } from "appwrite-utils";
 import { nameToIdMapping, processQueue } from "../migrations/queue.js";
 import { createUpdateCollectionAttributes } from "./attributes.js";
 import { createOrUpdateIndexes } from "./indexes.js";
-import _, { initial } from "lodash";
 import { SchemaGenerator } from "../migrations/schemaStrings.js";
+import {
+  isNull,
+  isUndefined,
+  isNil,
+  isPlainObject,
+  isString,
+  isJSONValue,
+} from "es-toolkit";
 import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 export const documentExists = async (
@@ -20,46 +27,50 @@ export const documentExists = async (
   targetCollectionId: string,
   toCreateObject: any
 ): Promise<Models.Document | null> => {
-  // Had to do this because kept running into issues with type checking arrays so, sorry 40ms
   const collection = await db.getCollection(dbId, targetCollectionId);
   const attributes = collection.attributes as any[];
   let arrayTypeAttributes = attributes
     .filter((attribute: any) => attribute.array === true)
     .map((attribute: any) => attribute.key);
-  // Function to check if a string is JSON
+
   const isJsonString = (str: string) => {
     try {
       const json = JSON.parse(str);
-      return typeof json === "object" && json !== null; // Check if parsed JSON is an object or array
+      return typeof json === "object" && json !== null;
     } catch (e) {
       return false;
     }
   };
 
-  // Validate and prepare query parameters
-  const validQueryParams = _.chain(toCreateObject)
-    .pickBy(
-      (value, key) =>
-        !arrayTypeAttributes.includes(key) &&
-        !key.startsWith("$") &&
-        !_.isNull(value) &&
-        !_.isUndefined(value) &&
-        !_.isEmpty(value) &&
-        !_.isObject(value) && // Keeps excluding objects
-        !_.isArray(value) && // Explicitly exclude arrays
-        !(_.isString(value) && isJsonString(value)) && // Exclude JSON strings
-        (_.isString(value) ? value.length < 4096 && value.length > 0 : true) // String length check
-    )
-    .mapValues((value, key) =>
-      _.isString(value) || _.isNumber(value) || _.isBoolean(value)
+  // Convert object to entries and filter
+  const validEntries = Object.entries(toCreateObject).filter(
+    ([key, value]) =>
+      !arrayTypeAttributes.includes(key) &&
+      !key.startsWith("$") &&
+      !isNull(value) &&
+      !isUndefined(value) &&
+      !isNil(value) &&
+      !isPlainObject(value) &&
+      !Array.isArray(value) &&
+      !(isString(value) && isJsonString(value)) &&
+      (isString(value) ? value.length < 4096 && value.length > 0 : true)
+  );
+
+  // Map and filter valid entries
+  const validMappedEntries = validEntries
+    .map(([key, value]) => [
+      key,
+      isString(value) || typeof value === "number" || typeof value === "boolean"
         ? value
-        : null
-    )
-    .omitBy(_.isNull) // Remove any null values that might have been added in mapValues
-    .toPairs()
-    .slice(0, 25) // Limit to 25 to adhere to query limit
-    .map(([key, value]) => Query.equal(key, value as any))
-    .value();
+        : null,
+    ])
+    .filter(([key, value]) => !isNull(value) && isString(key))
+    .slice(0, 25);
+
+  // Convert to Query parameters
+  const validQueryParams = validMappedEntries.map(([key, value]) =>
+    Query.equal(key as string, value as any)
+  );
 
   // Execute the query with the validated and prepared parameters
   const result = await db.listDocuments(
@@ -380,12 +391,18 @@ export const createOrUpdateCollections = async (
     // Add delay after creating attributes
     await delay(250);
 
+    const indexesToUse =
+      indexes.length > 0
+        ? indexes
+        : config.collections?.find((c) => c.$id === collectionToUse!.$id)
+            ?.indexes ?? [];
+
     console.log("Creating Indexes");
     await createOrUpdateIndexes(
       databaseId,
       database,
       collectionToUse!.$id,
-      (indexes ?? []) as Indexes
+      indexesToUse as Indexes
     );
 
     // Add delay after creating indexes
@@ -444,3 +461,217 @@ export const fetchAllCollections = async (
   console.log(`Fetched a total of ${collections.length} collections.`);
   return collections;
 };
+
+/**
+ * Transfers all documents from one collection to another in a different database
+ * within the same Appwrite Project
+ */
+export const transferDocumentsBetweenDbsLocalToLocal = async (
+  db: Databases,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  let fromCollDocs = await tryAwaitWithRetry(async () =>
+    db.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
+  );
+  let totalDocumentsTransferred = 0;
+
+  if (fromCollDocs.documents.length === 0) {
+    console.log(`No documents found in collection ${fromCollId}`);
+    return;
+  } else if (fromCollDocs.documents.length < 50) {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: Partial<typeof doc> = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(
+        async () =>
+          await db.createDocument(
+            toDbId,
+            toCollId,
+            doc.$id,
+            toCreateObject,
+            doc.$permissions
+          )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+  } else {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: Partial<typeof doc> = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(async () =>
+        db.createDocument(
+          toDbId,
+          toCollId,
+          doc.$id,
+          toCreateObject,
+          doc.$permissions
+        )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+    while (fromCollDocs.documents.length === 50) {
+      fromCollDocs = await tryAwaitWithRetry(
+        async () =>
+          await db.listDocuments(fromDbId, fromCollId, [
+            Query.limit(50),
+            Query.cursorAfter(
+              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
+            ),
+          ])
+      );
+      const batchedPromises = fromCollDocs.documents.map((doc) => {
+        const toCreateObject: Partial<typeof doc> = {
+          ...doc,
+        };
+        delete toCreateObject.$databaseId;
+        delete toCreateObject.$collectionId;
+        delete toCreateObject.$createdAt;
+        delete toCreateObject.$updatedAt;
+        delete toCreateObject.$id;
+        delete toCreateObject.$permissions;
+        return tryAwaitWithRetry(
+          async () =>
+            await db.createDocument(
+              toDbId,
+              toCollId,
+              doc.$id,
+              toCreateObject,
+              doc.$permissions
+            )
+        );
+      });
+      await Promise.all(batchedPromises);
+      totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+  }
+
+  console.log(
+    `Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`
+  );
+};
+
+export const transferDocumentsBetweenDbsLocalToRemote = async (
+  localDb: Databases,
+  endpoint: string,
+  projectId: string,
+  apiKey: string,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  const client = new Client()
+    .setEndpoint(endpoint)
+    .setProject(projectId)
+    .setKey(apiKey);
+  let totalDocumentsTransferred = 0;
+  const remoteDb = new Databases(client);
+  let fromCollDocs = await tryAwaitWithRetry(async () =>
+    localDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
+  );
+
+  if (fromCollDocs.documents.length === 0) {
+    console.log(`No documents found in collection ${fromCollId}`);
+    return;
+  } else if (fromCollDocs.documents.length < 50) {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: Partial<typeof doc> = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(async () =>
+        remoteDb.createDocument(
+          toDbId,
+          toCollId,
+          doc.$id,
+          toCreateObject,
+          doc.$permissions
+        )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+  } else {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: Partial<typeof doc> = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(async () =>
+        remoteDb.createDocument(
+          toDbId,
+          toCollId,
+          doc.$id,
+          toCreateObject,
+          doc.$permissions
+        )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+    while (fromCollDocs.documents.length === 50) {
+      fromCollDocs = await tryAwaitWithRetry(async () =>
+        localDb.listDocuments(fromDbId, fromCollId, [
+          Query.limit(50),
+          Query.cursorAfter(
+            fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
+          ),
+        ])
+      );
+      const batchedPromises = fromCollDocs.documents.map((doc) => {
+        const toCreateObject: Partial<typeof doc> = {
+          ...doc,
+        };
+        delete toCreateObject.$databaseId;
+        delete toCreateObject.$collectionId;
+        delete toCreateObject.$createdAt;
+        delete toCreateObject.$updatedAt;
+        delete toCreateObject.$id;
+        delete toCreateObject.$permissions;
+        return tryAwaitWithRetry(async () =>
+          remoteDb.createDocument(
+            toDbId,
+            toCollId,
+            doc.$id,
+            toCreateObject,
+            doc.$permissions
+          )
+        );
+      });
+      await Promise.all(batchedPromises);
+      totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+  }
+  console.log(
+    `Total documents transferred from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}: ${totalDocumentsTransferred}`
+  );
+};

package/src/migrations/appwriteToX.ts

@@ -8,7 +8,7 @@ import {
   type Models,
   type Permission,
 } from "node-appwrite";
-import { fetchAllCollections } from "./collections.js";
+import { fetchAllCollections } from "../collections/methods.js";
 import { fetchAllDatabases } from "./databases.js";
 import {
   CollectionSchema,
@@ -138,7 +138,7 @@ export class AppwriteToX {
           .map((attr: any) => {
             return parseAttribute(attr);
           })
-          .filter((attribute) =>
+          .filter((attribute: Attribute) =>
             attribute.type === "relationship"
               ? attribute.side !== "child"
               : true
@@ -169,10 +169,10 @@ export class AppwriteToX {
           }
         }
         this.collToAttributeMap.set(collection.name, collAttributes);
-        const finalIndexes = collection.indexes.map((index) => {
+        const finalIndexes = collection.indexes.map((index: Models.Index) => {
           return {
             ...index,
-            orders: index.orders?.filter((order) => {
+            orders: index.orders?.filter((order: string) => {
               return order !== null && order;
             }),
           };

package/src/migrations/attributes.ts

@@ -5,7 +5,6 @@ import {
   type Attribute,
 } from "appwrite-utils";
 import { nameToIdMapping, enqueueOperation } from "./queue.js";
-import _ from "lodash";
 import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 const attributesSame = (

package/src/migrations/converters.ts

@@ -1,7 +1,5 @@
-import _ from "lodash";
 import { converterFunctions, type AttributeMappings } from "appwrite-utils";
-
-const { cloneDeep, isObject } = _;
+import { cloneDeep, isPlainObject } from "es-toolkit";
 
 /**
  * Deeply converts all properties of an object (or array) to strings.
@@ -11,7 +9,7 @@ const { cloneDeep, isObject } = _;
 export const deepAnyToString = (data: any): any => {
   if (Array.isArray(data)) {
     return data.map((item) => deepAnyToString(item));
-  } else if (isObject(data)) {
+  } else if (isPlainObject(data)) {
     return Object.keys(data).reduce((acc, key) => {
       acc[key] = deepAnyToString(data[key as keyof typeof data]);
       return acc;
@@ -34,7 +32,7 @@ export const deepConvert = <T>(
 ): any => {
   if (Array.isArray(data)) {
     return data.map((item) => deepConvert(item, convertFn));
-  } else if (isObject(data)) {
+  } else if (isPlainObject(data)) {
     return Object.keys(data).reduce((acc: Record<string, T>, key: string) => {
       acc[key] = deepConvert(data[key as keyof typeof data], convertFn);
       return acc;

package/src/migrations/dataLoader.ts

@@ -16,14 +16,14 @@ import path from "path";
 import fs from "fs";
 import { convertObjectByAttributeMappings } from "./converters.js";
 import { z } from "zod";
-import { checkForCollection } from "./collections.js";
+import { checkForCollection } from "../collections/methods.js";
 import { ID, Users, type Databases } from "node-appwrite";
 import { logger } from "./logging.js";
 import { findOrCreateOperation, updateOperation } from "./migrationHelper.js";
 import { AuthUserCreateSchema } from "../schemas/authUser.js";
-import _ from "lodash";
 import { UsersController } from "./users.js";
 import { finalizeByAttributeMap } from "../utils/helperFunctions.js";
+import { isEmpty } from "es-toolkit/compat";
 
 // Define a schema for the structure of collection import data using Zod for validation
 export const CollectionImportDataSchema = z.object({
@@ -315,7 +315,7 @@ export class DataLoader {
         // Find or create an import operation for the collection
         const collectionImportOperation = await findOrCreateOperation(
           this.database,
-          collection.$id,
+          collection.$id!,
           "importData"
         );
         // Store the operation ID in the map
@@ -556,7 +556,7 @@ export class DataLoader {
                 // Skip if value to match is missing or empty
                 if (
                   !sourceValue ||
-                  _.isEmpty(sourceValue) ||
+                  isEmpty(sourceValue) ||
                   sourceValue === null
                 )
                   continue;

package/src/migrations/importController.ts

@@ -13,7 +13,6 @@ import type {
   AttributeMappings,
 } from "appwrite-utils";
 import type { ImportDataActions } from "./importDataActions.js";
-import _ from "lodash";
 import { areCollectionNamesSame, tryAwaitWithRetry } from "../utils/index.js";
 import type { SetupOptions } from "../utilsController.js";
 import { resolveAndUpdateRelationships } from "./relationships.js";

package/src/migrations/queue.ts

@@ -1,8 +1,7 @@
 import { Query, type Databases, type Models } from "node-appwrite";
 import type { Attribute } from "appwrite-utils";
 import { createOrUpdateAttribute } from "./attributes.js";
-import _ from "lodash";
-import { fetchAndCacheCollectionByName } from "./collections.js";
+import { fetchAndCacheCollectionByName } from "../collections/methods.js";
 import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 export interface QueuedOperation {

package/src/migrations/relationships.ts

@@ -1,5 +1,5 @@
 import { Databases, Query, type Models } from "node-appwrite";
-import { fetchAllCollections } from "./collections.js";
+import { fetchAllCollections } from "../collections/methods.js";
 import type {
   AppwriteConfig,
   Attribute,

package/src/migrations/users.ts

@@ -12,13 +12,14 @@ import {
   type AuthUser,
   type AuthUserCreate,
 } from "../schemas/authUser.js";
-import _ from "lodash";
 import { logger } from "./logging.js";
 import { splitIntoBatches } from "./migrationHelper.js";
 import {
   getAppwriteClient,
   tryAwaitWithRetry,
 } from "../utils/helperFunctions.js";
+import { isUndefined } from "es-toolkit/compat";
+import { isEmpty } from "es-toolkit/compat";
 
 export class UsersController {
   private config: AppwriteConfig;
@@ -123,17 +124,17 @@ export class UsersController {
           `changeMe${item.email?.toLowerCase()}` || `changeMePlease`,
           item.name || undefined
         );
-  
+
         if (item.labels) {
           await this.users.updateLabels(createdUser.$id, item.labels);
         }
         if (item.prefs) {
           await this.users.updatePrefs(createdUser.$id, item.prefs);
         }
-        
+
         return createdUser;
       }); // Set throwError to true since we want to handle errors
-  
+
       return user;
     } catch (e) {
       if (e instanceof Error) {
@@ -181,8 +182,8 @@ export class UsersController {
         if (
           item.email &&
           item.email !== userToReturn.email &&
-          !_.isEmpty(item.email) &&
-          !_.isUndefined(item.email)
+          !isEmpty(item.email) &&
+          !isUndefined(item.email)
         ) {
           const emailExists = await this.users.list([
             Query.equal("email", item.email),
@@ -213,7 +214,7 @@ export class UsersController {
           item.phone !== userToReturn.phone &&
           item.phone.length < 15 &&
           item.phone.startsWith("+") &&
-          (_.isUndefined(userToReturn.phone) || _.isEmpty(userToReturn.phone))
+          (isUndefined(userToReturn.phone) || isEmpty(userToReturn.phone))
         ) {
           const userFoundWithPhone = await this.users.list([
             Query.equal("phone", item.phone),

package/src/migrations/validationRules.ts

@@ -1,17 +1,42 @@
-import _ from "lodash";
-
+import {
+  isNumber,
+  isString,
+  isBoolean,
+  isArray,
+  isPlainObject,
+  isNull,
+  isUndefined,
+  isDate,
+  isEmpty,
+  isInteger,
+  isArrayLike,
+  isArrayLikeObject,
+  isFunction,
+  isLength,
+  isMap,
+  isSet,
+  isRegExp,
+  isSymbol,
+  isObjectLike,
+  isSafeInteger,
+  isTypedArray,
+  isEqual,
+  isMatch,
+  has,
+  get,
+} from "es-toolkit/compat";
 export interface ValidationRules {
   [key: string]: (value: any, ...args: any[]) => boolean;
 }
 
 export const validationRules = {
-  isNumber: (value: any): boolean => _.isNumber(value),
-  isString: (value: any): boolean => _.isString(value),
-  isBoolean: (value: any): boolean => _.isBoolean(value),
-  isArray: (value: any): boolean => _.isArray(value),
+  isNumber: (value: any): boolean => isNumber(value),
+  isString: (value: any): boolean => isString(value),
+  isBoolean: (value: any): boolean => isBoolean(value),
+  isArray: (value: any): boolean => isArray(value),
   isObject: (value: any): boolean =>
-    _.isObject(value) && !_.isArray(value) && !_.isFunction(value),
-  isNull: (value: any): boolean => _.isNull(value),
+    isPlainObject(value) && !isArray(value) && !isFunction(value),
+  isNull: (value: any): boolean => isNull(value),
   isValidEmail: (value: string): boolean =>
     value.match(/^[\w\-\.]+@([\w-]+\.)+[\w-]{2,}$/) !== null,
   isValidPhone: (value: string): boolean =>
@@ -35,29 +60,29 @@ export const validationRules = {
     value.match(/^\d{4}-\d{2}-\d{2}$/) !== null,
   isValidTime: (value: string): boolean =>
     value.match(/^\d{2}:\d{2}(:\d{2})?$/) !== null,
-  isNullish: (value: any): boolean => _.isNull(value) || _.isUndefined(value),
-  isUndefined: (value: any): boolean => _.isUndefined(value),
+  isNullish: (value: any): boolean => isNull(value) || isUndefined(value),
+  isUndefined: (value: any): boolean => isUndefined(value),
   isDefined: (value: any): boolean =>
-    !_.isUndefined(value) && !_.isNull(value) && !_.isEmpty(value),
-  isDate: (value: any): boolean => _.isDate(value),
-  isEmpty: (value: any): boolean => _.isEmpty(value),
-  isInteger: (value: any): boolean => _.isInteger(value),
-  isFloat: (value: any): boolean => _.isNumber(value) && !_.isInteger(value),
-  isArrayLike: (value: any): boolean => _.isArrayLike(value),
-  isArrayLikeObject: (value: any): boolean => _.isArrayLikeObject(value),
-  isFunction: (value: any): boolean => _.isFunction(value),
-  isLength: (value: any): boolean => _.isLength(value),
-  isMap: (value: any): boolean => _.isMap(value),
-  isSet: (value: any): boolean => _.isSet(value),
-  isRegExp: (value: any): boolean => _.isRegExp(value),
-  isSymbol: (value: any): boolean => _.isSymbol(value),
-  isObjectLike: (value: any): boolean => _.isObjectLike(value),
-  isPlainObject: (value: any): boolean => _.isPlainObject(value),
-  isSafeInteger: (value: any): boolean => _.isSafeInteger(value),
-  isTypedArray: (value: any): boolean => _.isTypedArray(value),
-  isEqual: (value: any, other: any): boolean => _.isEqual(value, other),
-  isMatch: (object: any, source: any): boolean => _.isMatch(object, source),
-  has: (object: any, path: string): boolean => _.has(object, path),
+    !isUndefined(value) && !isNull(value) && !isEmpty(value),
+  isDate: (value: any): boolean => isDate(value),
+  isEmpty: (value: any): boolean => isEmpty(value),
+  isInteger: (value: any): boolean => isInteger(value),
+  isFloat: (value: any): boolean => isNumber(value) && !isInteger(value),
+  isArrayLike: (value: any): boolean => isArrayLike(value),
+  isArrayLikeObject: (value: any): boolean => isArrayLikeObject(value),
+  isFunction: (value: any): boolean => isFunction(value),
+  isLength: (value: any): boolean => isLength(value),
+  isMap: (value: any): boolean => isMap(value),
+  isSet: (value: any): boolean => isSet(value),
+  isRegExp: (value: any): boolean => isRegExp(value),
+  isSymbol: (value: any): boolean => isSymbol(value),
+  isObjectLike: (value: any): boolean => isObjectLike(value),
+  isPlainObject: (value: any): boolean => isPlainObject(value),
+  isSafeInteger: (value: any): boolean => isSafeInteger(value),
+  isTypedArray: (value: any): boolean => isTypedArray(value),
+  isEqual: (value: any, other: any): boolean => isEqual(value, other),
+  isMatch: (object: any, source: any): boolean => isMatch(object, source),
+  has: (object: any, path: string): boolean => has(object, path),
   get: (object: any, path: string, defaultValue: any): any =>
-    _.get(object, path, defaultValue),
+    get(object, path, defaultValue),
 };