appwrite-utils-cli 0.0.37 → 0.0.39

package/README.md

@@ -86,6 +86,7 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 ### Changelog
 
+- 0.0.38: Lots of optimizations done to the code, added `tryAwaitWithRetry` for `fetch failed` and others like it errors (looking at you `server error`) -- this should prevent things from going sideways.
 - 0.0.37: Added `documentSecurity`, `enabled`, and `$id` to the `init` collection
 - 0.0.36: Made it update collections by default, sometimes you gotta do what you gotta do
 - 0.0.35: Added update collection if it exists and permissions or such are different (`documentSecurity` and `enabled`), also added a check for `fetch failed` errors to retry them with recursion, not sure how well that will work out, but we're gonna try it! It will still fail after 5 tries, but hopefully that gives Appwrite some time to figure it's stuff out

package/dist/migrations/attributes.js

@@ -2,6 +2,7 @@ import { Query } from "node-appwrite";
 import { attributeSchema, parseAttribute, } from "appwrite-utils";
 import { nameToIdMapping, enqueueOperation } from "./queue.js";
 import _ from "lodash";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 const attributesSame = (databaseAttribute, configAttribute) => {
     return (databaseAttribute.key == configAttribute.key &&
         databaseAttribute.type == configAttribute.type &&
@@ -105,10 +106,10 @@ export const createOrUpdateAttribute = async (db, dbId, collection, attribute) =
     switch (finalAttribute.type) {
         case "string":
             if (action === "create") {
-                await db.createStringAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.size, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array || false, finalAttribute.encrypted);
+                await tryAwaitWithRetry(async () => await db.createStringAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.size, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array || false, finalAttribute.encrypted));
             }
             else {
-                await db.updateStringAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined);
+                await tryAwaitWithRetry(async () => await db.updateStringAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined));
             }
             break;
         case "integer":
@@ -121,7 +122,7 @@ export const createOrUpdateAttribute = async (db, dbId, collection, attribute) =
                     BigInt(finalAttribute.max) === BigInt(9223372036854776000)) {
                     delete finalAttribute.max;
                 }
-                await db.createIntegerAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.min, finalAttribute.max, finalAttribute.xdefault || undefined, finalAttribute.array);
+                await tryAwaitWithRetry(async () => await db.createIntegerAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.min, finalAttribute.max, finalAttribute.xdefault || undefined, finalAttribute.array));
             }
             else {
                 if (finalAttribute.min &&
@@ -132,71 +133,71 @@ export const createOrUpdateAttribute = async (db, dbId, collection, attribute) =
                     BigInt(finalAttribute.max) === BigInt(9223372036854776000)) {
                     delete finalAttribute.max;
                 }
-                await db.updateIntegerAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.min || 0, finalAttribute.max || 2147483647, finalAttribute.xdefault || undefined);
+                await tryAwaitWithRetry(async () => await db.updateIntegerAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.min || 0, finalAttribute.max || 2147483647, finalAttribute.xdefault || undefined));
             }
             break;
         case "float":
             if (action === "create") {
-                await db.createFloatAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.min, finalAttribute.max, finalAttribute.xdefault || undefined, finalAttribute.array);
+                await tryAwaitWithRetry(async () => await db.createFloatAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.min, finalAttribute.max, finalAttribute.xdefault || undefined, finalAttribute.array));
             }
             else {
-                await db.updateFloatAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.min || 0, finalAttribute.max || 2147483647, finalAttribute.xdefault || undefined);
+                await tryAwaitWithRetry(async () => await db.updateFloatAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.min || 0, finalAttribute.max || 2147483647, finalAttribute.xdefault || undefined));
             }
             break;
         case "boolean":
             if (action === "create") {
-                await db.createBooleanAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array);
+                await tryAwaitWithRetry(async () => await db.createBooleanAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array));
             }
             else {
-                await db.updateBooleanAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || null);
+                await tryAwaitWithRetry(async () => await db.updateBooleanAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || null));
             }
             break;
         case "datetime":
             if (action === "create") {
-                await db.createDatetimeAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array);
+                await tryAwaitWithRetry(async () => await db.createDatetimeAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array));
             }
             else {
-                await db.updateDatetimeAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined);
+                await tryAwaitWithRetry(async () => await db.updateDatetimeAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined));
             }
             break;
         case "email":
             if (action === "create") {
-                await db.createEmailAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array);
+                await tryAwaitWithRetry(async () => await db.createEmailAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array));
             }
             else {
-                await db.updateEmailAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined);
+                await tryAwaitWithRetry(async () => await db.updateEmailAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined));
             }
             break;
         case "ip":
             if (action === "create") {
-                await db.createIpAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array);
+                await tryAwaitWithRetry(async () => await db.createIpAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array));
             }
             else {
-                await db.updateIpAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined);
+                await tryAwaitWithRetry(async () => await db.updateIpAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined));
             }
             break;
         case "url":
             if (action === "create") {
-                await db.createUrlAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array);
+                await tryAwaitWithRetry(async () => await db.createUrlAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array));
             }
             else {
-                await db.updateUrlAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined);
+                await tryAwaitWithRetry(async () => await db.updateUrlAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.required || false, finalAttribute.xdefault || undefined));
             }
             break;
         case "enum":
             if (action === "create") {
-                await db.createEnumAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.elements, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array);
+                await tryAwaitWithRetry(async () => await db.createEnumAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.elements, finalAttribute.required || false, finalAttribute.xdefault || undefined, finalAttribute.array));
             }
             else {
-                await db.updateEnumAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.elements, finalAttribute.required || false, finalAttribute.xdefault || undefined);
+                await tryAwaitWithRetry(async () => await db.updateEnumAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.elements, finalAttribute.required || false, finalAttribute.xdefault || undefined));
             }
             break;
         case "relationship":
             if (action === "create") {
-                await db.createRelationshipAttribute(dbId, collection.$id, relatedCollectionId, finalAttribute.relationType, finalAttribute.twoWay, finalAttribute.key, finalAttribute.twoWayKey, finalAttribute.onDelete);
+                await tryAwaitWithRetry(async () => await db.createRelationshipAttribute(dbId, collection.$id, relatedCollectionId, finalAttribute.relationType, finalAttribute.twoWay, finalAttribute.key, finalAttribute.twoWayKey, finalAttribute.onDelete));
             }
             else {
-                await db.updateRelationshipAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.onDelete);
+                await tryAwaitWithRetry(async () => await db.updateRelationshipAttribute(dbId, collection.$id, finalAttribute.key, finalAttribute.onDelete));
             }
             break;
         default:

package/dist/migrations/collections.js

@@ -4,6 +4,7 @@ import { createUpdateCollectionAttributes } from "./attributes.js";
 import { createOrUpdateIndexes } from "./indexes.js";
 import _ from "lodash";
 import { SchemaGenerator } from "./schemaStrings.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export const documentExists = async (db, dbId, targetCollectionId, toCreateObject) => {
     // Had to do this because kept running into issues with type checking arrays so, sorry 40ms
     const collection = await db.getCollection(dbId, targetCollectionId);
@@ -48,9 +49,7 @@ export const documentExists = async (db, dbId, targetCollectionId, toCreateObjec
 export const checkForCollection = async (db, dbId, collection) => {
     try {
         console.log(`Checking for collection with name: ${collection.name}`);
-        const response = await db.listCollections(dbId, [
-            Query.equal("name", collection.name),
-        ]);
+        const response = await tryAwaitWithRetry(async () => await db.listCollections(dbId, [Query.equal("name", collection.name)]));
         if (response.collections.length > 0) {
             console.log(`Collection found: ${response.collections[0].$id}`);
             return { ...collection, ...response.collections[0] };
@@ -70,13 +69,11 @@ export const fetchAndCacheCollectionByName = async (db, dbId, collectionName) =>
     if (nameToIdMapping.has(collectionName)) {
         const collectionId = nameToIdMapping.get(collectionName);
         console.log(`\tCollection found in cache: ${collectionId}`);
-        return await db.getCollection(dbId, collectionId);
+        return await tryAwaitWithRetry(async () => await db.getCollection(dbId, collectionId));
     }
     else {
         console.log(`\tFetching collection by name: ${collectionName}`);
-        const collectionsPulled = await db.listCollections(dbId, [
-            Query.equal("name", collectionName),
-        ]);
+        const collectionsPulled = await tryAwaitWithRetry(async () => await db.listCollections(dbId, [Query.equal("name", collectionName)]));
         if (collectionsPulled.total > 0) {
             const collection = collectionsPulled.collections[0];
             console.log(`\tCollection found: ${collection.$id}`);
@@ -91,7 +88,7 @@ export const fetchAndCacheCollectionByName = async (db, dbId, collectionName) =>
 };
 export const wipeDatabase = async (database, databaseId) => {
     console.log(`Wiping database: ${databaseId}`);
-    const { collections: existingCollections } = await database.listCollections(databaseId);
+    const { collections: existingCollections } = await tryAwaitWithRetry(async () => await database.listCollections(databaseId));
     let collectionsDeleted = [];
     for (const { $id: collectionId, name: name } of existingCollections) {
         console.log(`Deleting collection: ${collectionId}`);
@@ -141,9 +138,9 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
             }
         }
         // Check if the collection already exists by name
-        let collectionsFound = await database.listCollections(databaseId, [
+        let collectionsFound = await tryAwaitWithRetry(async () => await database.listCollections(databaseId, [
             Query.equal("name", collection.name),
-        ]);
+        ]));
         let collectionToUse = collectionsFound.total > 0 ? collectionsFound.collections[0] : null;
         // Determine the correct ID for the collection
         let collectionId;
@@ -163,7 +160,7 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
             usedIds.add(collectionId); // Mark this ID as used
             // Create the collection with the determined ID
             try {
-                collectionToUse = await database.createCollection(databaseId, collectionId, collection.name, permissions, collection.documentSecurity ?? false, collection.enabled ?? true);
+                collectionToUse = await tryAwaitWithRetry(async () => await database.createCollection(databaseId, collectionId, collection.name, permissions, collection.documentSecurity ?? false, collection.enabled ?? true));
                 collection.$id = collectionToUse.$id;
                 nameToIdMapping.set(collection.name, collectionToUse.$id);
             }
@@ -174,7 +171,7 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
         }
         else {
             console.log(`Collection ${collection.name} exists, updating it`);
-            await database.updateCollection(databaseId, collectionToUse.$id, collection.name, permissions, collection.documentSecurity ?? false, collection.enabled ?? true);
+            await tryAwaitWithRetry(async () => await database.updateCollection(databaseId, collectionToUse.$id, collection.name, permissions, collection.documentSecurity ?? false, collection.enabled ?? true));
         }
         // Update attributes and indexes for the collection
         console.log("Creating Attributes");
@@ -206,7 +203,7 @@ export const fetchAllCollections = async (dbId, database) => {
         if (lastCollectionId) {
             queries.push(Query.cursorAfter(lastCollectionId));
         }
-        const response = await database.listCollections(dbId, queries);
+        const response = await tryAwaitWithRetry(async () => await database.listCollections(dbId, queries));
         collections = collections.concat(response.collections);
         moreCollections = response.collections.length === 500;
         if (moreCollections) {

package/dist/migrations/converters.js

@@ -94,7 +94,12 @@ export const convertObjectByAttributeMappings = (obj, attributeMappings) => {
             const values = mapping.oldKeys
                 .map((oldKey) => resolveValue(obj, oldKey))
                 .flat(Infinity);
-            result[mapping.targetKey] = values.filter((value) => value !== undefined);
+            if (values.length > 0) {
+                result[mapping.targetKey] = values.filter((value) => value !== undefined);
+            }
+            else {
+                result[mapping.targetKey] = null;
+            }
         }
         else if (mapping.oldKey) {
             // Resolve single oldKey
@@ -104,6 +109,15 @@ export const convertObjectByAttributeMappings = (obj, attributeMappings) => {
                     ? value.flat(Infinity)
                     : value;
             }
+            else {
+                result[mapping.targetKey] = null;
+            }
+        }
+    }
+    // Ensure any keys in the original object that are not mapped are copied over
+    for (const key of Object.keys(obj)) {
+        if (!Object.keys(result).includes(key)) {
+            result[key] = obj[key];
         }
     }
     return result;

package/dist/migrations/dataLoader.d.ts

@@ -2,6 +2,7 @@ import type { ImportDataActions } from "./importDataActions.js";
 import { type AppwriteConfig, type AttributeMappings, type CollectionCreate, type ConfigDatabase, type ImportDef } from "appwrite-utils";
 import { z } from "zod";
 import { type Databases } from "node-appwrite";
+import { AuthUserCreateSchema } from "../schemas/authUser.js";
 export declare const CollectionImportDataSchema: z.ZodObject<{
     collection: z.ZodOptional<z.ZodObject<Omit<{
         name: z.ZodString;
@@ -61,14 +62,6 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             required: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
             array: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
             min: z.ZodOptional<z.ZodNumber>;
-            /**
-             * Prepares the data for creating documents in a collection.
-             * This involves loading the data, transforming it, and handling ID mappings.
-             *
-             * @param db - The database configuration.
-             * @param collection - The collection configuration.
-             * @param importDef - The import definition containing the attribute mappings and other relevant info.
-             */
             max: z.ZodOptional<z.ZodNumber>;
             xdefault: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
             description: z.ZodOptional<z.ZodNullable<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodString>]>>>;
@@ -367,16 +360,6 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             type: z.ZodOptional<z.ZodDefault<z.ZodEnum<["create", "update"]>>>;
             filePath: z.ZodString;
             basePath: z.ZodOptional<z.ZodString>;
-            /**
-             * Generates attribute mappings with post-import actions based on the provided attribute mappings.
-             * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
-             * and update the field with the file's ID if necessary.
-             *
-             * @param attributeMappings - The attribute mappings from the import definition.
-             * @param context - The context object containing information about the database, collection, and document.
-             * @param item - The item being imported, used for resolving template paths in fileData mappings.
-             * @returns The attribute mappings updated with any necessary post-import actions.
-             */
             primaryKeyField: z.ZodDefault<z.ZodString>;
             idMappings: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 sourceField: z.ZodString;
@@ -1713,6 +1696,7 @@ export declare class DataLoader {
      * It iterates through the target object's keys and updates the source object if:
      * - The source object has the key.
      * - The target object's value for that key is not null, undefined, or an empty string.
+     * - If the target object has an array value, it concatenates the values and removes duplicates.
      *
      * @param source - The source object to be updated.
      * @param target - The target object with values to update the source object.
@@ -1736,7 +1720,7 @@ export declare class DataLoader {
     getAllUsers(): Promise<import("node-appwrite").Models.User<import("node-appwrite").Models.Preferences>[]>;
     start(dbId: string): Promise<void>;
     dealWithMergedUsers(): void;
-    updateOldReferencesForNew(): Promise<void>;
+    updateOldReferencesForNew(): void;
     private writeMapsToJsonFile;
     /**
      * Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,
@@ -1746,7 +1730,14 @@ export declare class DataLoader {
      * @param attributeMappings - The attribute mappings for the item.
      * @returns The transformed item with user-specific keys removed.
      */
-    prepareUserData(item: any, attributeMappings: AttributeMappings, primaryKeyField: string, newId: string): Promise<any>;
+    prepareUserData(item: any, attributeMappings: AttributeMappings, primaryKeyField: string, newId: string): {
+        transformedItem: any;
+        existingId: string | undefined;
+        userData: {
+            rawData: any;
+            finalData: z.infer<typeof AuthUserCreateSchema>;
+        };
+    };
     /**
      * Prepares the data for creating user collection documents.
      * This involves loading the data, transforming it according to the import definition,
@@ -1776,7 +1767,7 @@ export declare class DataLoader {
      * @param collection - The collection configuration.
      * @param importDef - The import definition containing the attribute mappings and other relevant info.
      */
-    prepareUpdateData(db: ConfigDatabase, collection: CollectionCreate, importDef: ImportDef): Promise<void>;
+    prepareUpdateData(db: ConfigDatabase, collection: CollectionCreate, importDef: ImportDef): void;
     private updateReferencesBasedOnAttributeMappings;
     private getMergedId;
     /**