appwrite-utils-cli 0.0.49 → 0.0.52

package/README.md

@@ -132,6 +132,7 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 ### Changelog
 
+- 0.0.50: Actually fixed the slight bug, it was really in the `mergeObjects`
 - 0.0.49: Fixed a slight bug with `dataLoader` not mapping updates correctly with `updateMapping`
 - 0.0.48: Added `--transfer`, `--fromdb <targetDatabaseId>`, `--targetdb <targetDatabaseId>`, `--transferendpoint <transferEndpoint>`, `--transferproject <transferProjectId>`, `--transferkey <transferApiKey>`. Additionally, I've added `--fromcoll <collectionId>` and `--targetcoll <collectionId>`. These allow you to do a few things. First, you can now transfer databases in the same project, and from local to a remote project. Second, you can now specify specific collections to transfer from one place to another, with all of their data. If `--fromcoll` and `--targetcoll` are ommitted, it will transfer the databases. During the database transfer, it will create any missing collections, attributes, and indices.
 - 0.0.47: Minor bugfixes in many releases, too small to take note of

package/dist/migrations/collections.js

@@ -257,10 +257,10 @@ export const transferDocumentsBetweenDbsLocalToLocal = async (db, fromDbId, toDb
         await Promise.all(batchedPromises);
         totalDocumentsTransferred += fromCollDocs.documents.length;
         while (fromCollDocs.documents.length === 50) {
-            fromCollDocs = await db.listDocuments(fromDbId, fromCollId, [
+            fromCollDocs = await tryAwaitWithRetry(async () => await db.listDocuments(fromDbId, fromCollId, [
                 Query.limit(50),
                 Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
-            ]);
+            ]));
             const batchedPromises = fromCollDocs.documents.map((doc) => {
                 const toCreateObject = {
                     ...doc,

package/dist/migrations/dataLoader.d.ts

@@ -192,7 +192,14 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             type: z.ZodLiteral<"ip">;
             error: z.ZodOptional<z.ZodDefault<z.ZodString>>;
             required: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
-            array: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
+            array: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>; /**
+             * Prepares the data for creating documents in a collection.
+             * This involves loading the data, transforming it, and handling ID mappings.
+             *
+             * @param db - The database configuration.
+             * @param collection - The collection configuration.
+             * @param importDef - The import definition containing the attribute mappings and other relevant info.
+             */
             xdefault: z.ZodOptional<z.ZodNullable<z.ZodString>>;
             description: z.ZodOptional<z.ZodNullable<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodString>]>>>;
         }, "strip", z.ZodTypeAny, {
@@ -278,16 +285,6 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 originalIdField: z.ZodString;
                 targetField: z.ZodOptional<z.ZodString>;
             }, "strip", z.ZodTypeAny, {
-                /**
-                 * Prepares the data for updating documents within a collection.
-                 * This method loads the raw data based on the import definition, transforms it according to the attribute mappings,
-                 * finds the new ID for each item based on the primary key or update mapping, and then validates the transformed data.
-                 * If the data is valid, it updates the import definition with any post-import actions and adds the item to the current collection data.
-                 *
-                 * @param db - The database configuration.
-                 * @param collection - The collection configuration.
-                 * @param importDef - The import definition containing the attribute mappings and other relevant info.
-                 */
                 originalIdField: string;
                 targetField?: string | undefined;
             }, {

package/dist/migrations/dataLoader.js

@@ -119,6 +119,26 @@ export class DataLoader {
                 continue;
             }
         }
+        // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
+        for (const [key, value] of Object.entries(update)) {
+            if (value === undefined || value === null || value === "") {
+                continue;
+            }
+            else if (!Object.hasOwn(source, key)) {
+                result[key] = value;
+            }
+            else if (typeof source[key] === "object" && typeof value === "object") {
+                result[key] = this.mergeObjects(source[key], value);
+            }
+            else if (Array.isArray(source[key]) && Array.isArray(value)) {
+                result[key] = [...new Set([...source[key], ...value])].filter((item) => item !== null && item !== undefined && item !== "");
+            }
+            else if (source[key] === undefined ||
+                source[key] === null ||
+                source[key] === "") {
+                result[key] = value;
+            }
+        }
         return result;
     }
     // Method to load data from a file specified in the import definition
@@ -314,7 +334,7 @@ export class DataLoader {
                 }
             }
             console.log("Running update references");
-            this.dealWithMergedUsers();
+            // this.dealWithMergedUsers();
             this.updateOldReferencesForNew();
             console.log("Done running update references");
         }

package/dist/migrations/migrationHelper.js

@@ -67,18 +67,18 @@ export const getAfterImportOperations = async (database, collectionId) => {
     return allOps;
 };
 export const findOrCreateOperation = async (database, collectionId, operationType, additionalQueries) => {
-    const operations = await database.listDocuments("migrations", "currentOperations", [
+    const operations = await tryAwaitWithRetry(async () => await database.listDocuments("migrations", "currentOperations", [
         Query.equal("collectionId", collectionId),
         Query.equal("operationType", operationType),
         Query.equal("status", "pending"),
         ...(additionalQueries || []),
-    ]);
+    ]));
     if (operations.documents.length > 0) {
         return OperationSchema.parse(operations.documents[0]); // Assuming the first document is the operation we want
     }
     else {
         // Create a new operation document
-        const op = await database.createDocument("migrations", "currentOperations", ID.unique(), {
+        const op = await tryAwaitWithRetry(async () => await database.createDocument("migrations", "currentOperations", ID.unique(), {
             operationType,
             collectionId,
             status: "pending",
@@ -86,7 +86,7 @@ export const findOrCreateOperation = async (database, collectionId, operationTyp
             progress: 0,
             total: 0,
             error: "",
-        });
+        }));
         return OperationSchema.parse(op);
     }
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.49",
+  "version": "0.0.52",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/collections.ts

@@ -399,12 +399,15 @@ export const transferDocumentsBetweenDbsLocalToLocal = async (
     await Promise.all(batchedPromises);
     totalDocumentsTransferred += fromCollDocs.documents.length;
     while (fromCollDocs.documents.length === 50) {
-      fromCollDocs = await db.listDocuments(fromDbId, fromCollId, [
-        Query.limit(50),
-        Query.cursorAfter(
-          fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
-        ),
-      ]);
+      fromCollDocs = await tryAwaitWithRetry(
+        async () =>
+          await db.listDocuments(fromDbId, fromCollId, [
+            Query.limit(50),
+            Query.cursorAfter(
+              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
+            ),
+          ])
+      );
       const batchedPromises = fromCollDocs.documents.map((doc) => {
         const toCreateObject: Partial<typeof doc> = {
           ...doc,

package/src/migrations/dataLoader.ts

@@ -150,6 +150,26 @@ export class DataLoader {
         continue;
       }
     }
+    // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
+    for (const [key, value] of Object.entries(update)) {
+      if (value === undefined || value === null || value === "") {
+        continue;
+      } else if (!Object.hasOwn(source, key)) {
+        result[key] = value;
+      } else if (typeof source[key] === "object" && typeof value === "object") {
+        result[key] = this.mergeObjects(source[key], value);
+      } else if (Array.isArray(source[key]) && Array.isArray(value)) {
+        result[key] = [...new Set([...source[key], ...value])].filter(
+          (item) => item !== null && item !== undefined && item !== ""
+        );
+      } else if (
+        source[key] === undefined ||
+        source[key] === null ||
+        source[key] === ""
+      ) {
+        result[key] = value;
+      }
+    }
 
     return result;
   }
@@ -393,7 +413,7 @@ export class DataLoader {
         }
       }
       console.log("Running update references");
-      this.dealWithMergedUsers();
+      // this.dealWithMergedUsers();
       this.updateOldReferencesForNew();
       console.log("Done running update references");
     }

package/src/migrations/migrationHelper.ts

@@ -110,34 +110,36 @@ export const findOrCreateOperation = async (
   operationType: string,
   additionalQueries?: string[]
 ) => {
-  const operations = await database.listDocuments(
-    "migrations",
-    "currentOperations",
-    [
-      Query.equal("collectionId", collectionId),
-      Query.equal("operationType", operationType),
-      Query.equal("status", "pending"),
-      ...(additionalQueries || []),
-    ]
+  const operations = await tryAwaitWithRetry(
+    async () =>
+      await database.listDocuments("migrations", "currentOperations", [
+        Query.equal("collectionId", collectionId),
+        Query.equal("operationType", operationType),
+        Query.equal("status", "pending"),
+        ...(additionalQueries || []),
+      ])
   );
 
   if (operations.documents.length > 0) {
     return OperationSchema.parse(operations.documents[0]); // Assuming the first document is the operation we want
   } else {
     // Create a new operation document
-    const op = await database.createDocument(
-      "migrations",
-      "currentOperations",
-      ID.unique(),
-      {
-        operationType,
-        collectionId,
-        status: "pending",
-        batches: [],
-        progress: 0,
-        total: 0,
-        error: "",
-      }
+    const op = await tryAwaitWithRetry(
+      async () =>
+        await database.createDocument(
+          "migrations",
+          "currentOperations",
+          ID.unique(),
+          {
+            operationType,
+            collectionId,
+            status: "pending",
+            batches: [],
+            progress: 0,
+            total: 0,
+            error: "",
+          }
+        )
     );
 
     return OperationSchema.parse(op);