appwrite-utils-cli 0.0.54 → 0.0.55

package/README.md

@@ -132,6 +132,7 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 ### Changelog
 
+- 0.0.55: Added `documentExists` check to batch creation functionality to try to prevent duplicates
 - 0.0.54: Various fixes in here
 - 0.0.50: Actually fixed the slight bug, it was really in the `mergeObjects`
 - 0.0.49: Fixed a slight bug with `dataLoader` not mapping updates correctly with `updateMapping`

package/dist/migrations/importController.js

@@ -7,7 +7,7 @@ import { logger } from "./logging.js";
 import { updateOperation } from "./migrationHelper.js";
 import { BatchSchema, OperationCreateSchema, OperationSchema, } from "./backup.js";
 import { DataLoader } from "./dataLoader.js";
-import { transferDocumentsBetweenDbsLocalToLocal } from "./collections.js";
+import { documentExists, transferDocumentsBetweenDbsLocalToLocal, } from "./collections.js";
 import { transferDatabaseLocalToLocal } from "./databases.js";
 import { transferStorageLocalToLocal } from "./storage.js";
 export class ImportController {
@@ -166,7 +166,24 @@ export class ImportController {
             for (let i = 0; i < dataSplit.length; i++) {
                 const batches = dataSplit[i];
                 console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
-                const batchPromises = batches.map((item) => {
+                const documentExistsPromises = batches.map(async (item) => {
+                    try {
+                        const id = item.finalData.docId ||
+                            item.finalData.userId ||
+                            item.context.docId ||
+                            item.context.userId;
+                        if (!item.finalData) {
+                            return Promise.resolve(null);
+                        }
+                        return tryAwaitWithRetry(async () => await documentExists(this.database, db.$id, collection.$id, item.finalData));
+                    }
+                    catch (error) {
+                        console.error(error);
+                        return Promise.resolve(null);
+                    }
+                });
+                const documentExistsResults = await Promise.all(documentExistsPromises);
+                const batchPromises = batches.map((item, index) => {
                     try {
                         const id = item.finalData.docId ||
                             item.finalData.userId ||
@@ -178,7 +195,7 @@ export class ImportController {
                         if (item.finalData.hasOwnProperty("docId")) {
                             delete item.finalData.docId;
                         }
-                        if (!item.finalData) {
+                        if (!item.finalData || documentExistsResults[index]) {
                             return Promise.resolve();
                         }
                         return tryAwaitWithRetry(async () => await this.database.createDocument(db.$id, collection.$id, id, item.finalData));

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.54",
+  "version": "0.0.55",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/importController.ts

@@ -26,7 +26,10 @@ import {
   OperationSchema,
 } from "./backup.js";
 import { DataLoader, type CollectionImportData } from "./dataLoader.js";
-import { transferDocumentsBetweenDbsLocalToLocal } from "./collections.js";
+import {
+  documentExists,
+  transferDocumentsBetweenDbsLocalToLocal,
+} from "./collections.js";
 import { transferDatabaseLocalToLocal } from "./databases.js";
 import { transferStorageLocalToLocal } from "./storage.js";
 
@@ -244,20 +247,50 @@ export class ImportController {
       for (let i = 0; i < dataSplit.length; i++) {
         const batches = dataSplit[i];
         console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
-        const batchPromises = batches.map((item) => {
+
+        const documentExistsPromises = batches.map(async (item) => {
           try {
             const id =
               item.finalData.docId ||
               item.finalData.userId ||
               item.context.docId ||
               item.context.userId;
+
+            if (!item.finalData) {
+              return Promise.resolve(null);
+            }
+            return tryAwaitWithRetry(
+              async () =>
+                await documentExists(
+                  this.database,
+                  db.$id,
+                  collection.$id,
+                  item.finalData
+                )
+            );
+          } catch (error) {
+            console.error(error);
+            return Promise.resolve(null);
+          }
+        });
+
+        const documentExistsResults = await Promise.all(documentExistsPromises);
+
+        const batchPromises = batches.map((item, index) => {
+          try {
+            const id =
+              item.finalData.docId ||
+              item.finalData.userId ||
+              item.context.docId ||
+              item.context.userId;
+
             if (item.finalData.hasOwnProperty("userId")) {
               delete item.finalData.userId;
             }
             if (item.finalData.hasOwnProperty("docId")) {
               delete item.finalData.docId;
             }
-            if (!item.finalData) {
+            if (!item.finalData || documentExistsResults[index]) {
               return Promise.resolve();
             }
             return tryAwaitWithRetry(
@@ -274,6 +307,7 @@ export class ImportController {
             return Promise.resolve();
           }
         });
+
         // Wait for all promises in the current batch to resolve
         await Promise.all(batchPromises);
         console.log(`Completed batch ${i + 1} of ${dataSplit.length}`);