appwrite-utils-cli 0.9.5 → 0.9.51

package/README.md

@@ -124,6 +124,7 @@ This updated CLI ensures that developers have robust tools at their fingertips t
 
 ## Changelog
 
+- 0.9.51: Fix transfer databases, remove "ensure duplicates" check
 - 0.9.5: Fixed not checking for storage bucket for each database (checking the creation status) when importing data
 - 0.9.4: Fixed migrations database ensuring it has the required collections
 - 0.9.3: Fixed deployment error && fix lack of existing `appwriteConfig.ts` file from causing error (you want to be able to setup yeah? haha)

package/dist/interactiveCLI.js

@@ -416,14 +416,14 @@ export class InteractiveCLI {
                 default: false,
             },
         ]);
-        const { checkDuplicates } = await inquirer.prompt([
-            {
-                type: "confirm",
-                name: "checkDuplicates",
-                message: "Do you want to check for duplicates during import?",
-                default: true,
-            },
-        ]);
+        // const { checkDuplicates } = await inquirer.prompt([
+        //   {
+        //     type: "confirm",
+        //     name: "checkDuplicates",
+        //     message: "Do you want to check for duplicates during import?",
+        //     default: true,
+        //   },
+        // ]);
         console.log("Importing data...");
         await this.controller.importData({
             databases: selectedDatabases,
@@ -431,7 +431,8 @@ export class InteractiveCLI {
                 ? selectedCollections.map((c) => c.$id)
                 : undefined,
             shouldWriteFile,
-            checkDuplicates,
+            checkDuplicates: false,
+            // checkDuplicates,
         });
     }
     async transferData() {
@@ -476,10 +477,21 @@ export class InteractiveCLI {
         }
         else {
             targetClient = sourceClient;
-            sourceDatabases = targetDatabases = await fetchAllDatabases(sourceClient);
+            const allDatabases = await fetchAllDatabases(sourceClient);
+            sourceDatabases = targetDatabases = allDatabases;
+        }
+        const fromDbs = await this.selectDatabases(sourceDatabases, "Select the source database:", false);
+        console.log(fromDbs);
+        const fromDb = fromDbs;
+        if (!fromDb) {
+            throw new Error("No source database selected");
+        }
+        const availableDbs = targetDatabases.filter((db) => db.$id !== fromDb.$id);
+        const targetDbs = await this.selectDatabases(availableDbs, "Select the target database:", false);
+        const targetDb = targetDbs;
+        if (!targetDb) {
+            throw new Error("No target database selected");
         }
-        const [fromDb] = await this.selectDatabases(sourceDatabases, "Select the source database:", false);
-        const [targetDb] = await this.selectDatabases(targetDatabases.filter((db) => db.$id !== fromDb.$id), "Select the target database:", false);
         const selectedCollections = await this.selectCollections(fromDb, sourceClient, "Select collections to transfer):");
         const { transferStorage } = await inquirer.prompt([
             {
@@ -499,8 +511,10 @@ export class InteractiveCLI {
             const targetBuckets = isRemote
                 ? await listBuckets(targetStorage)
                 : sourceBuckets;
-            [sourceBucket] = await this.selectBuckets(sourceBuckets.buckets, "Select the source bucket:", false);
-            [targetBucket] = await this.selectBuckets(targetBuckets.buckets, "Select the target bucket:", false);
+            const sourceBucketPicked = await this.selectBuckets(sourceBuckets.buckets, "Select the source bucket:", false);
+            const targetBucketPicked = await this.selectBuckets(targetBuckets.buckets, "Select the target bucket:", false);
+            sourceBucket = sourceBucketPicked;
+            targetBucket = targetBucketPicked;
         }
         let transferOptions = {
             fromDb,

package/dist/migrations/transfer.js

@@ -100,14 +100,26 @@ export const transferStorageLocalToRemote = async (localStorage, endpoint, proje
  * within the same Appwrite Project
  */
 export const transferDocumentsBetweenDbsLocalToLocal = async (db, fromDbId, toDbId, fromCollId, toCollId) => {
-    let fromCollDocs = await tryAwaitWithRetry(async () => db.listDocuments(fromDbId, fromCollId, [Query.limit(50)]));
     let totalDocumentsTransferred = 0;
-    if (fromCollDocs.documents.length === 0) {
-        console.log(`No documents found in collection ${fromCollId}`);
-        return;
-    }
-    else if (fromCollDocs.documents.length < 50) {
-        const batchedPromises = fromCollDocs.documents.map((doc) => {
+    let lastDocumentId;
+    let hasMoreDocuments = true;
+    while (hasMoreDocuments) {
+        const queryParams = [Query.limit(50)];
+        if (lastDocumentId) {
+            queryParams.push(Query.cursorAfter(lastDocumentId));
+        }
+        const fromCollDocs = await tryAwaitWithRetry(async () => db.listDocuments(fromDbId, fromCollId, queryParams));
+        if (fromCollDocs.documents.length === 0) {
+            if (totalDocumentsTransferred === 0) {
+                console.log(`No documents found in collection ${fromCollId}`);
+            }
+            break;
+        }
+        const allDocsToCreateCheck = await tryAwaitWithRetry(async () => await db.listDocuments(toDbId, toCollId, [
+            Query.equal("$id", fromCollDocs.documents.map((doc) => doc.$id)),
+        ]));
+        const docsToCreate = fromCollDocs.documents.filter((doc) => !allDocsToCreateCheck.documents.some((d) => d.$id === doc.$id));
+        const batchedPromises = docsToCreate.map((doc) => {
             const toCreateObject = {
                 ...doc,
             };
@@ -120,42 +132,13 @@ export const transferDocumentsBetweenDbsLocalToLocal = async (db, fromDbId, toDb
             return tryAwaitWithRetry(async () => await db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
         });
         await Promise.all(batchedPromises);
-        totalDocumentsTransferred += fromCollDocs.documents.length;
-    }
-    else {
-        const batchedPromises = fromCollDocs.documents.map((doc) => {
-            const toCreateObject = {
-                ...doc,
-            };
-            delete toCreateObject.$databaseId;
-            delete toCreateObject.$collectionId;
-            delete toCreateObject.$createdAt;
-            delete toCreateObject.$updatedAt;
-            delete toCreateObject.$id;
-            delete toCreateObject.$permissions;
-            return tryAwaitWithRetry(async () => db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
-        });
-        await Promise.all(batchedPromises);
-        totalDocumentsTransferred += fromCollDocs.documents.length;
-        while (fromCollDocs.documents.length === 50) {
-            fromCollDocs = await tryAwaitWithRetry(async () => await db.listDocuments(fromDbId, fromCollId, [
-                Query.limit(50),
-                Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
-            ]));
-            const batchedPromises = fromCollDocs.documents.map((doc) => {
-                const toCreateObject = {
-                    ...doc,
-                };
-                delete toCreateObject.$databaseId;
-                delete toCreateObject.$collectionId;
-                delete toCreateObject.$createdAt;
-                delete toCreateObject.$updatedAt;
-                delete toCreateObject.$id;
-                delete toCreateObject.$permissions;
-                return tryAwaitWithRetry(async () => await db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
-            });
-            await Promise.all(batchedPromises);
-            totalDocumentsTransferred += fromCollDocs.documents.length;
+        totalDocumentsTransferred += docsToCreate.length;
+        if (fromCollDocs.documents.length < 50) {
+            hasMoreDocuments = false;
+        }
+        else {
+            lastDocumentId =
+                fromCollDocs.documents[fromCollDocs.documents.length - 1].$id;
         }
     }
     console.log(`Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`);
@@ -173,7 +156,11 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (localDb, endpoint
         return;
     }
     else if (fromCollDocs.documents.length < 50) {
-        const batchedPromises = fromCollDocs.documents.map((doc) => {
+        const allDocsToCreateCheck = await tryAwaitWithRetry(async () => await remoteDb.listDocuments(toDbId, toCollId, [
+            Query.equal("$id", fromCollDocs.documents.map((doc) => doc.$id)),
+        ]));
+        const docsToCreate = fromCollDocs.documents.filter((doc) => !allDocsToCreateCheck.documents.some((d) => d.$id === doc.$id));
+        const batchedPromises = docsToCreate.map((doc) => {
             const toCreateObject = {
                 ...doc,
             };
@@ -189,7 +176,11 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (localDb, endpoint
         totalDocumentsTransferred += fromCollDocs.documents.length;
     }
     else {
-        const batchedPromises = fromCollDocs.documents.map((doc) => {
+        const allDocsToCreateCheck = await tryAwaitWithRetry(async () => await remoteDb.listDocuments(toDbId, toCollId, [
+            Query.equal("$id", fromCollDocs.documents.map((doc) => doc.$id)),
+        ]));
+        const docsToCreate = fromCollDocs.documents.filter((doc) => !allDocsToCreateCheck.documents.some((d) => d.$id === doc.$id));
+        const batchedPromises = docsToCreate.map((doc) => {
             const toCreateObject = {
                 ...doc,
             };
@@ -305,9 +296,7 @@ export const transferDatabaseLocalToRemote = async (localDb, endpoint, projectId
     let lastCollectionId;
     let fromCollections = await tryAwaitWithRetry(async () => await localDb.listCollections(fromDbId, [Query.limit(50)]));
     const allFromCollections = fromCollections.collections;
-    if (fromCollections.collections.length < 50) {
-    }
-    else {
+    if (fromCollections.collections.length >= 50) {
         lastCollectionId =
             fromCollections.collections[fromCollections.collections.length - 1].$id;
         while (lastCollectionId) {
@@ -324,13 +313,58 @@ export const transferDatabaseLocalToRemote = async (localDb, endpoint, projectId
         }
     }
     for (const collection of allFromCollections) {
-        const toCollection = await tryAwaitWithRetry(async () => await remoteDb.createCollection(toDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
-        console.log(`Collection ${toCollection.name} created`);
+        let toCollection;
+        const toCollectionExists = await tryAwaitWithRetry(async () => await remoteDb.listCollections(toDbId, [
+            Query.equal("$id", collection.$id),
+        ]));
+        if (toCollectionExists.collections.length > 0) {
+            console.log(`Collection ${collection.name} already exists. Updating...`);
+            toCollection = toCollectionExists.collections[0];
+            // Update collection if needed
+            if (toCollection.name !== collection.name ||
+                toCollection.$permissions !== collection.$permissions ||
+                toCollection.documentSecurity !== collection.documentSecurity ||
+                toCollection.enabled !== collection.enabled) {
+                toCollection = await tryAwaitWithRetry(async () => await remoteDb.updateCollection(toDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
+                console.log(`Collection ${toCollection.name} updated`);
+            }
+        }
+        else {
+            toCollection = await tryAwaitWithRetry(async () => await remoteDb.createCollection(toDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
+            console.log(`Collection ${toCollection.name} created`);
+        }
+        // Check and update attributes
+        const existingAttributes = await tryAwaitWithRetry(async () => await remoteDb.listAttributes(toDbId, toCollection.$id));
         for (const attribute of collection.attributes) {
-            await tryAwaitWithRetry(async () => await createOrUpdateAttribute(remoteDb, toDbId, toCollection, parseAttribute(attribute)));
+            const parsedAttribute = parseAttribute(attribute);
+            const existingAttribute = existingAttributes.attributes.find(
+            // @ts-expect-error
+            (attr) => attr.key === parsedAttribute.key);
+            if (!existingAttribute) {
+                await tryAwaitWithRetry(async () => await createOrUpdateAttribute(remoteDb, toDbId, toCollection, parsedAttribute));
+                console.log(`Attribute ${parsedAttribute.key} created`);
+            }
+            else {
+                // Check if attribute needs updating
+                // Note: Appwrite doesn't allow updating most attribute properties
+                // You might need to delete and recreate the attribute if significant changes are needed
+                console.log(`Attribute ${parsedAttribute.key} already exists`);
+            }
         }
+        // Check and update indexes
+        const existingIndexes = await tryAwaitWithRetry(async () => await remoteDb.listIndexes(toDbId, toCollection.$id));
         for (const index of collection.indexes) {
-            await tryAwaitWithRetry(async () => await remoteDb.createIndex(toDbId, toCollection.$id, index.key, index.type, index.attributes, index.orders));
+            const existingIndex = existingIndexes.indexes.find((idx) => idx.key === index.key);
+            if (!existingIndex) {
+                await tryAwaitWithRetry(async () => await remoteDb.createIndex(toDbId, toCollection.$id, index.key, index.type, index.attributes, index.orders));
+                console.log(`Index ${index.key} created`);
+            }
+            else {
+                // Check if index needs updating
+                // Note: Appwrite doesn't allow updating indexes
+                // You might need to delete and recreate the index if changes are needed
+                console.log(`Index ${index.key} already exists`);
+            }
         }
         await transferDocumentsBetweenDbsLocalToRemote(localDb, endpoint, projectId, apiKey, fromDbId, toDbId, collection.$id, toCollection.$id);
     }

package/package.json

@@ -1,53 +1,53 @@
-{
-  "name": "appwrite-utils-cli",
-  "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.9.5",
-  "main": "src/main.ts",
-  "type": "module",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/zachhandley/AppwriteUtils"
-  },
-  "author": "Zach Handley <zach@blackleafdigital.com> (https://zachhandley.com)",
-  "keywords": [
-    "appwrite",
-    "cli",
-    "utils",
-    "migrations",
-    "data",
-    "database",
-    "import",
-    "migration",
-    "utility"
-  ],
-  "bin": {
-    "appwrite-migrate": "./dist/main.js"
-  },
-  "scripts": {
-    "build": "bun run tsc",
-    "start": "tsx --no-cache src/main.ts",
-    "deploy": "bun run build && npm publish --access public",
-    "postinstall": "echo 'This package is intended for CLI use only and should not be added as a dependency in other projects.'"
-  },
-  "dependencies": {
-    "@types/inquirer": "^9.0.7",
-    "appwrite-utils": "^0.3.8",
-    "commander": "^12.1.0",
-    "inquirer": "^9.3.6",
-    "js-yaml": "^4.1.0",
-    "lodash": "^4.17.21",
-    "luxon": "^3.5.0",
-    "nanostores": "^0.10.3",
-    "node-appwrite": "^13.0.0",
-    "tsx": "^4.17.0",
-    "ulidx": "^2.4.0",
-    "winston": "^3.14.1",
-    "zod": "^3.23.8"
-  },
-  "devDependencies": {
-    "@types/js-yaml": "^4.0.9",
-    "@types/lodash": "^4.17.7",
-    "@types/luxon": "^3.4.2",
-    "typescript": "^5.5.4"
-  }
-}
+{
+  "name": "appwrite-utils-cli",
+  "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
+  "version": "0.9.51",
+  "main": "src/main.ts",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/zachhandley/AppwriteUtils"
+  },
+  "author": "Zach Handley <zach@blackleafdigital.com> (https://zachhandley.com)",
+  "keywords": [
+    "appwrite",
+    "cli",
+    "utils",
+    "migrations",
+    "data",
+    "database",
+    "import",
+    "migration",
+    "utility"
+  ],
+  "bin": {
+    "appwrite-migrate": "./dist/main.js"
+  },
+  "scripts": {
+    "build": "bun run tsc",
+    "start": "tsx --no-cache src/main.ts",
+    "deploy": "bun run build && npm publish --access public",
+    "postinstall": "echo 'This package is intended for CLI use only and should not be added as a dependency in other projects.'"
+  },
+  "dependencies": {
+    "@types/inquirer": "^9.0.7",
+    "appwrite-utils": "^0.3.8",
+    "commander": "^12.1.0",
+    "inquirer": "^9.3.6",
+    "js-yaml": "^4.1.0",
+    "lodash": "^4.17.21",
+    "luxon": "^3.5.0",
+    "nanostores": "^0.10.3",
+    "node-appwrite": "^13.0.0",
+    "tsx": "^4.17.0",
+    "ulidx": "^2.4.0",
+    "winston": "^3.14.2",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/lodash": "^4.17.7",
+    "@types/luxon": "^3.4.2",
+    "typescript": "^5.5.4"
+  }
+}

package/src/interactiveCLI.ts

@@ -558,14 +558,14 @@ export class InteractiveCLI {
       },
     ]);
 
-    const { checkDuplicates } = await inquirer.prompt([
-      {
-        type: "confirm",
-        name: "checkDuplicates",
-        message: "Do you want to check for duplicates during import?",
-        default: true,
-      },
-    ]);
+    // const { checkDuplicates } = await inquirer.prompt([
+    //   {
+    //     type: "confirm",
+    //     name: "checkDuplicates",
+    //     message: "Do you want to check for duplicates during import?",
+    //     default: true,
+    //   },
+    // ]);
 
     console.log("Importing data...");
     await this.controller!.importData({
@@ -575,7 +575,8 @@ export class InteractiveCLI {
           ? selectedCollections.map((c) => c.$id)
           : undefined,
       shouldWriteFile,
-      checkDuplicates,
+      checkDuplicates: false,
+      // checkDuplicates,
     });
   }
 
@@ -637,19 +638,42 @@ export class InteractiveCLI {
       targetDatabases = await fetchAllDatabases(targetClient);
     } else {
       targetClient = sourceClient;
-      sourceDatabases = targetDatabases = await fetchAllDatabases(sourceClient);
+      const allDatabases = await fetchAllDatabases(sourceClient);
+      sourceDatabases = targetDatabases = allDatabases;
     }
 
-    const [fromDb] = await this.selectDatabases(
+    const fromDbs = await this.selectDatabases(
       sourceDatabases,
       "Select the source database:",
       false
     );
-    const [targetDb] = await this.selectDatabases(
-      targetDatabases.filter((db) => db.$id !== fromDb.$id),
+    console.log(fromDbs);
+    const fromDb = fromDbs as unknown as {
+      $id: string;
+      name: string;
+      $createdAt: string;
+      $updatedAt: string;
+      enabled: boolean;
+    };
+    if (!fromDb) {
+      throw new Error("No source database selected");
+    }
+    const availableDbs = targetDatabases.filter((db) => db.$id !== fromDb.$id);
+    const targetDbs = await this.selectDatabases(
+      availableDbs,
       "Select the target database:",
       false
     );
+    const targetDb = targetDbs as unknown as {
+      $id: string;
+      name: string;
+      $createdAt: string;
+      $updatedAt: string;
+      enabled: boolean;
+    };
+    if (!targetDb) {
+      throw new Error("No target database selected");
+    }
 
     const selectedCollections = await this.selectCollections(
       fromDb,
@@ -685,16 +709,18 @@ export class InteractiveCLI {
         ? await listBuckets(targetStorage)
         : sourceBuckets;
 
-      [sourceBucket] = await this.selectBuckets(
+      const sourceBucketPicked = await this.selectBuckets(
         sourceBuckets.buckets,
         "Select the source bucket:",
         false
       );
-      [targetBucket] = await this.selectBuckets(
+      const targetBucketPicked = await this.selectBuckets(
         targetBuckets.buckets,
         "Select the target bucket:",
         false
       );
+      sourceBucket = sourceBucketPicked as unknown as Models.Bucket;
+      targetBucket = targetBucketPicked as unknown as Models.Bucket;
     }
 
     let transferOptions: TransferOptions = {