appwrite-utils-cli 0.0.48 → 0.0.51

package/README.md

@@ -132,6 +132,8 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 ### Changelog
 
+- 0.0.50: Actually fixed the slight bug, it was really in the `mergeObjects`
+- 0.0.49: Fixed a slight bug with `dataLoader` not mapping updates correctly with `updateMapping`
 - 0.0.48: Added `--transfer`, `--fromdb <targetDatabaseId>`, `--targetdb <targetDatabaseId>`, `--transferendpoint <transferEndpoint>`, `--transferproject <transferProjectId>`, `--transferkey <transferApiKey>`. Additionally, I've added `--fromcoll <collectionId>` and `--targetcoll <collectionId>`. These allow you to do a few things. First, you can now transfer databases in the same project, and from local to a remote project. Second, you can now specify specific collections to transfer from one place to another, with all of their data. If `--fromcoll` and `--targetcoll` are ommitted, it will transfer the databases. During the database transfer, it will create any missing collections, attributes, and indices.
 - 0.0.47: Minor bugfixes in many releases, too small to take note of
 - 0.0.38: Lots of optimizations done to the code, added `tryAwaitWithRetry` for `fetch failed` and others like it errors (looking at you `server error`) -- this should prevent things from going sideways.

package/dist/migrations/collections.js

@@ -257,10 +257,10 @@ export const transferDocumentsBetweenDbsLocalToLocal = async (db, fromDbId, toDb
         await Promise.all(batchedPromises);
         totalDocumentsTransferred += fromCollDocs.documents.length;
         while (fromCollDocs.documents.length === 50) {
-            fromCollDocs = await db.listDocuments(fromDbId, fromCollId, [
+            fromCollDocs = await tryAwaitWithRetry(async () => await db.listDocuments(fromDbId, fromCollId, [
                 Query.limit(50),
                 Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
-            ]);
+            ]));
             const batchedPromises = fromCollDocs.documents.map((doc) => {
                 const toCreateObject = {
                     ...doc,

package/dist/migrations/dataLoader.d.ts

@@ -278,16 +278,6 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 originalIdField: z.ZodString;
                 targetField: z.ZodOptional<z.ZodString>;
             }, "strip", z.ZodTypeAny, {
-                /**
-                 * Prepares the data for updating documents within a collection.
-                 * This method loads the raw data based on the import definition, transforms it according to the attribute mappings,
-                 * finds the new ID for each item based on the primary key or update mapping, and then validates the transformed data.
-                 * If the data is valid, it updates the import definition with any post-import actions and adds the item to the current collection data.
-                 *
-                 * @param db - The database configuration.
-                 * @param collection - The collection configuration.
-                 * @param importDef - The import definition containing the attribute mappings and other relevant info.
-                 */
                 originalIdField: string;
                 targetField?: string | undefined;
             }, {

package/dist/migrations/dataLoader.js

@@ -119,6 +119,26 @@ export class DataLoader {
                 continue;
             }
         }
+        // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
+        for (const [key, value] of Object.entries(update)) {
+            if (value === undefined || value === null || value === "") {
+                continue;
+            }
+            else if (!Object.hasOwn(source, key)) {
+                result[key] = value;
+            }
+            else if (typeof source[key] === "object" && typeof value === "object") {
+                result[key] = this.mergeObjects(source[key], value);
+            }
+            else if (Array.isArray(source[key]) && Array.isArray(value)) {
+                result[key] = [...new Set([...source[key], ...value])].filter((item) => item !== null && item !== undefined && item !== "");
+            }
+            else if (source[key] === undefined ||
+                source[key] === null ||
+                source[key] === "") {
+                result[key] = value;
+            }
+        }
         return result;
     }
     // Method to load data from a file specified in the import definition
@@ -878,7 +898,10 @@ export class DataLoader {
             let itemDataToUpdate;
             // Try to find itemDataToUpdate using updateMapping
             if (importDef.updateMapping) {
-                oldId = item[importDef.updateMapping.originalIdField];
+                console.log(importDef.updateMapping);
+                oldId =
+                    item[importDef.updateMapping.originalIdField] ||
+                        transformedData[importDef.updateMapping.originalIdField];
                 if (oldId) {
                     itemDataToUpdate = currentData?.data.find(({ context, rawData, finalData }) => {
                         const targetField = importDef.updateMapping.targetField ??
@@ -896,7 +919,9 @@ export class DataLoader {
             }
             // If updateMapping is not defined or did not find the item, use primaryKeyField
             if (!itemDataToUpdate && importDef.primaryKeyField) {
-                oldId = item[importDef.primaryKeyField];
+                oldId =
+                    item[importDef.primaryKeyField] ||
+                        transformedData[importDef.primaryKeyField];
                 if (oldId) {
                     newId = oldIdToNewIdMap?.get(`${oldId}`);
                     if (!newId &&
@@ -919,13 +944,14 @@ export class DataLoader {
                 continue;
             }
             if (!newId && !itemDataToUpdate) {
-                logger.error(`No new id found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
+                logger.error(`No new id && no data found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
                 continue;
             }
             else if (itemDataToUpdate) {
-                newId = itemDataToUpdate.finalData.docId;
+                newId =
+                    itemDataToUpdate.finalData.docId || itemDataToUpdate.context.docId;
                 if (!newId) {
-                    logger.error(`No new id found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
+                    logger.error(`No new id found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but has itemDataToUpdate ${JSON.stringify(itemDataToUpdate, null, 2)} but it says it's supposed to have one...`);
                     continue;
                 }
             }

package/dist/migrations/migrationHelper.js

@@ -67,18 +67,18 @@ export const getAfterImportOperations = async (database, collectionId) => {
     return allOps;
 };
 export const findOrCreateOperation = async (database, collectionId, operationType, additionalQueries) => {
-    const operations = await database.listDocuments("migrations", "currentOperations", [
+    const operations = await tryAwaitWithRetry(async () => await database.listDocuments("migrations", "currentOperations", [
         Query.equal("collectionId", collectionId),
         Query.equal("operationType", operationType),
         Query.equal("status", "pending"),
         ...(additionalQueries || []),
-    ]);
+    ]));
     if (operations.documents.length > 0) {
         return OperationSchema.parse(operations.documents[0]); // Assuming the first document is the operation we want
     }
     else {
         // Create a new operation document
-        const op = await database.createDocument("migrations", "currentOperations", ID.unique(), {
+        const op = await tryAwaitWithRetry(async () => await database.createDocument("migrations", "currentOperations", ID.unique(), {
             operationType,
             collectionId,
             status: "pending",
@@ -86,7 +86,7 @@ export const findOrCreateOperation = async (database, collectionId, operationTyp
             progress: 0,
             total: 0,
             error: "",
-        });
+        }));
         return OperationSchema.parse(op);
     }
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.48",
+  "version": "0.0.51",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/collections.ts

@@ -399,12 +399,15 @@ export const transferDocumentsBetweenDbsLocalToLocal = async (
     await Promise.all(batchedPromises);
     totalDocumentsTransferred += fromCollDocs.documents.length;
     while (fromCollDocs.documents.length === 50) {
-      fromCollDocs = await db.listDocuments(fromDbId, fromCollId, [
-        Query.limit(50),
-        Query.cursorAfter(
-          fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
-        ),
-      ]);
+      fromCollDocs = await tryAwaitWithRetry(
+        async () =>
+          await db.listDocuments(fromDbId, fromCollId, [
+            Query.limit(50),
+            Query.cursorAfter(
+              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
+            ),
+          ])
+      );
       const batchedPromises = fromCollDocs.documents.map((doc) => {
         const toCreateObject: Partial<typeof doc> = {
           ...doc,

package/src/migrations/dataLoader.ts

@@ -150,6 +150,26 @@ export class DataLoader {
         continue;
       }
     }
+    // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
+    for (const [key, value] of Object.entries(update)) {
+      if (value === undefined || value === null || value === "") {
+        continue;
+      } else if (!Object.hasOwn(source, key)) {
+        result[key] = value;
+      } else if (typeof source[key] === "object" && typeof value === "object") {
+        result[key] = this.mergeObjects(source[key], value);
+      } else if (Array.isArray(source[key]) && Array.isArray(value)) {
+        result[key] = [...new Set([...source[key], ...value])].filter(
+          (item) => item !== null && item !== undefined && item !== ""
+        );
+      } else if (
+        source[key] === undefined ||
+        source[key] === null ||
+        source[key] === ""
+      ) {
+        result[key] = value;
+      }
+    }
 
     return result;
   }
@@ -1207,13 +1227,17 @@ export class DataLoader {
 
       // Try to find itemDataToUpdate using updateMapping
       if (importDef.updateMapping) {
-        oldId = item[importDef.updateMapping.originalIdField];
+        console.log(importDef.updateMapping);
+        oldId =
+          item[importDef.updateMapping.originalIdField] ||
+          transformedData[importDef.updateMapping.originalIdField];
         if (oldId) {
           itemDataToUpdate = currentData?.data.find(
             ({ context, rawData, finalData }) => {
               const targetField =
                 importDef.updateMapping!.targetField ??
                 importDef.updateMapping!.originalIdField;
+
               return (
                 `${context[targetField]}` === `${oldId}` ||
                 `${rawData[targetField]}` === `${oldId}` ||
@@ -1232,7 +1256,9 @@ export class DataLoader {
 
       // If updateMapping is not defined or did not find the item, use primaryKeyField
       if (!itemDataToUpdate && importDef.primaryKeyField) {
-        oldId = item[importDef.primaryKeyField];
+        oldId =
+          item[importDef.primaryKeyField] ||
+          transformedData[importDef.primaryKeyField];
         if (oldId) {
           newId = oldIdToNewIdMap?.get(`${oldId}`);
           if (
@@ -1268,7 +1294,7 @@ export class DataLoader {
 
       if (!newId && !itemDataToUpdate) {
         logger.error(
-          `No new id found for collection ${
+          `No new id && no data found for collection ${
             collection.name
           } for updateDef ${JSON.stringify(
             item,
@@ -1278,7 +1304,8 @@ export class DataLoader {
         );
         continue;
       } else if (itemDataToUpdate) {
-        newId = itemDataToUpdate.finalData.docId;
+        newId =
+          itemDataToUpdate.finalData.docId || itemDataToUpdate.context.docId;
         if (!newId) {
           logger.error(
             `No new id found for collection ${
@@ -1287,6 +1314,10 @@ export class DataLoader {
               item,
               null,
               2
+            )} but has itemDataToUpdate ${JSON.stringify(
+              itemDataToUpdate,
+              null,
+              2
             )} but it says it's supposed to have one...`
           );
           continue;

package/src/migrations/migrationHelper.ts

@@ -110,34 +110,36 @@ export const findOrCreateOperation = async (
   operationType: string,
   additionalQueries?: string[]
 ) => {
-  const operations = await database.listDocuments(
-    "migrations",
-    "currentOperations",
-    [
-      Query.equal("collectionId", collectionId),
-      Query.equal("operationType", operationType),
-      Query.equal("status", "pending"),
-      ...(additionalQueries || []),
-    ]
+  const operations = await tryAwaitWithRetry(
+    async () =>
+      await database.listDocuments("migrations", "currentOperations", [
+        Query.equal("collectionId", collectionId),
+        Query.equal("operationType", operationType),
+        Query.equal("status", "pending"),
+        ...(additionalQueries || []),
+      ])
   );
 
   if (operations.documents.length > 0) {
     return OperationSchema.parse(operations.documents[0]); // Assuming the first document is the operation we want
   } else {
     // Create a new operation document
-    const op = await database.createDocument(
-      "migrations",
-      "currentOperations",
-      ID.unique(),
-      {
-        operationType,
-        collectionId,
-        status: "pending",
-        batches: [],
-        progress: 0,
-        total: 0,
-        error: "",
-      }
+    const op = await tryAwaitWithRetry(
+      async () =>
+        await database.createDocument(
+          "migrations",
+          "currentOperations",
+          ID.unique(),
+          {
+            operationType,
+            collectionId,
+            status: "pending",
+            batches: [],
+            progress: 0,
+            total: 0,
+            error: "",
+          }
+        )
     );
 
     return OperationSchema.parse(op);