appwrite-utils-cli 0.9.5 → 0.9.52

package/src/interactiveCLI.ts

@@ -558,14 +558,14 @@ export class InteractiveCLI {
       },
     ]);
 
-    const { checkDuplicates } = await inquirer.prompt([
-      {
-        type: "confirm",
-        name: "checkDuplicates",
-        message: "Do you want to check for duplicates during import?",
-        default: true,
-      },
-    ]);
+    // const { checkDuplicates } = await inquirer.prompt([
+    //   {
+    //     type: "confirm",
+    //     name: "checkDuplicates",
+    //     message: "Do you want to check for duplicates during import?",
+    //     default: true,
+    //   },
+    // ]);
 
     console.log("Importing data...");
     await this.controller!.importData({
@@ -575,7 +575,8 @@ export class InteractiveCLI {
           ? selectedCollections.map((c) => c.$id)
           : undefined,
       shouldWriteFile,
-      checkDuplicates,
+      checkDuplicates: false,
+      // checkDuplicates,
     });
   }
 
@@ -637,19 +638,42 @@ export class InteractiveCLI {
       targetDatabases = await fetchAllDatabases(targetClient);
     } else {
       targetClient = sourceClient;
-      sourceDatabases = targetDatabases = await fetchAllDatabases(sourceClient);
+      const allDatabases = await fetchAllDatabases(sourceClient);
+      sourceDatabases = targetDatabases = allDatabases;
     }
 
-    const [fromDb] = await this.selectDatabases(
+    const fromDbs = await this.selectDatabases(
       sourceDatabases,
       "Select the source database:",
       false
     );
-    const [targetDb] = await this.selectDatabases(
-      targetDatabases.filter((db) => db.$id !== fromDb.$id),
+    console.log(fromDbs);
+    const fromDb = fromDbs as unknown as {
+      $id: string;
+      name: string;
+      $createdAt: string;
+      $updatedAt: string;
+      enabled: boolean;
+    };
+    if (!fromDb) {
+      throw new Error("No source database selected");
+    }
+    const availableDbs = targetDatabases.filter((db) => db.$id !== fromDb.$id);
+    const targetDbs = await this.selectDatabases(
+      availableDbs,
       "Select the target database:",
       false
     );
+    const targetDb = targetDbs as unknown as {
+      $id: string;
+      name: string;
+      $createdAt: string;
+      $updatedAt: string;
+      enabled: boolean;
+    };
+    if (!targetDb) {
+      throw new Error("No target database selected");
+    }
 
     const selectedCollections = await this.selectCollections(
       fromDb,
@@ -685,16 +709,18 @@ export class InteractiveCLI {
         ? await listBuckets(targetStorage)
         : sourceBuckets;
 
-      [sourceBucket] = await this.selectBuckets(
+      const sourceBucketPicked = await this.selectBuckets(
         sourceBuckets.buckets,
         "Select the source bucket:",
         false
       );
-      [targetBucket] = await this.selectBuckets(
+      const targetBucketPicked = await this.selectBuckets(
         targetBuckets.buckets,
         "Select the target bucket:",
         false
       );
+      sourceBucket = sourceBucketPicked as unknown as Models.Bucket;
+      targetBucket = targetBucketPicked as unknown as Models.Bucket;
     }
 
     let transferOptions: TransferOptions = {

package/src/migrations/transfer.ts

@@ -194,16 +194,42 @@ export const transferDocumentsBetweenDbsLocalToLocal = async (
   fromCollId: string,
   toCollId: string
 ) => {
-  let fromCollDocs = await tryAwaitWithRetry(async () =>
-    db.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
-  );
   let totalDocumentsTransferred = 0;
+  let lastDocumentId: string | undefined;
+  let hasMoreDocuments = true;
 
-  if (fromCollDocs.documents.length === 0) {
-    console.log(`No documents found in collection ${fromCollId}`);
-    return;
-  } else if (fromCollDocs.documents.length < 50) {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
+  while (hasMoreDocuments) {
+    const queryParams = [Query.limit(50)];
+    if (lastDocumentId) {
+      queryParams.push(Query.cursorAfter(lastDocumentId));
+    }
+
+    const fromCollDocs = await tryAwaitWithRetry(async () =>
+      db.listDocuments(fromDbId, fromCollId, queryParams)
+    );
+
+    if (fromCollDocs.documents.length === 0) {
+      if (totalDocumentsTransferred === 0) {
+        console.log(`No documents found in collection ${fromCollId}`);
+      }
+      break;
+    }
+
+    const allDocsToCreateCheck = await tryAwaitWithRetry(
+      async () =>
+        await db.listDocuments(toDbId, toCollId, [
+          Query.equal(
+            "$id",
+            fromCollDocs.documents.map((doc) => doc.$id)
+          ),
+        ])
+    );
+
+    const docsToCreate = fromCollDocs.documents.filter(
+      (doc) => !allDocsToCreateCheck.documents.some((d) => d.$id === doc.$id)
+    );
+
+    const batchedPromises = docsToCreate.map((doc) => {
       const toCreateObject: Partial<typeof doc> = {
         ...doc,
       };
@@ -224,64 +250,15 @@ export const transferDocumentsBetweenDbsLocalToLocal = async (
           )
       );
     });
+
     await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
-  } else {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
-      const toCreateObject: Partial<typeof doc> = {
-        ...doc,
-      };
-      delete toCreateObject.$databaseId;
-      delete toCreateObject.$collectionId;
-      delete toCreateObject.$createdAt;
-      delete toCreateObject.$updatedAt;
-      delete toCreateObject.$id;
-      delete toCreateObject.$permissions;
-      return tryAwaitWithRetry(async () =>
-        db.createDocument(
-          toDbId,
-          toCollId,
-          doc.$id,
-          toCreateObject,
-          doc.$permissions
-        )
-      );
-    });
-    await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
-    while (fromCollDocs.documents.length === 50) {
-      fromCollDocs = await tryAwaitWithRetry(
-        async () =>
-          await db.listDocuments(fromDbId, fromCollId, [
-            Query.limit(50),
-            Query.cursorAfter(
-              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
-            ),
-          ])
-      );
-      const batchedPromises = fromCollDocs.documents.map((doc) => {
-        const toCreateObject: Partial<typeof doc> = {
-          ...doc,
-        };
-        delete toCreateObject.$databaseId;
-        delete toCreateObject.$collectionId;
-        delete toCreateObject.$createdAt;
-        delete toCreateObject.$updatedAt;
-        delete toCreateObject.$id;
-        delete toCreateObject.$permissions;
-        return tryAwaitWithRetry(
-          async () =>
-            await db.createDocument(
-              toDbId,
-              toCollId,
-              doc.$id,
-              toCreateObject,
-              doc.$permissions
-            )
-        );
-      });
-      await Promise.all(batchedPromises);
-      totalDocumentsTransferred += fromCollDocs.documents.length;
+    totalDocumentsTransferred += docsToCreate.length;
+
+    if (fromCollDocs.documents.length < 50) {
+      hasMoreDocuments = false;
+    } else {
+      lastDocumentId =
+        fromCollDocs.documents[fromCollDocs.documents.length - 1].$id;
     }
   }
 
@@ -314,7 +291,19 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
     console.log(`No documents found in collection ${fromCollId}`);
     return;
   } else if (fromCollDocs.documents.length < 50) {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
+    const allDocsToCreateCheck = await tryAwaitWithRetry(
+      async () =>
+        await remoteDb.listDocuments(toDbId, toCollId, [
+          Query.equal(
+            "$id",
+            fromCollDocs.documents.map((doc) => doc.$id)
+          ),
+        ])
+    );
+    const docsToCreate = fromCollDocs.documents.filter(
+      (doc) => !allDocsToCreateCheck.documents.some((d) => d.$id === doc.$id)
+    );
+    const batchedPromises = docsToCreate.map((doc) => {
       const toCreateObject: Partial<typeof doc> = {
         ...doc,
       };
@@ -337,7 +326,19 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
     await Promise.all(batchedPromises);
     totalDocumentsTransferred += fromCollDocs.documents.length;
   } else {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
+    const allDocsToCreateCheck = await tryAwaitWithRetry(
+      async () =>
+        await remoteDb.listDocuments(toDbId, toCollId, [
+          Query.equal(
+            "$id",
+            fromCollDocs.documents.map((doc) => doc.$id)
+          ),
+        ])
+    );
+    const docsToCreate = fromCollDocs.documents.filter(
+      (doc) => !allDocsToCreateCheck.documents.some((d) => d.$id === doc.$id)
+    );
+    const batchedPromises = docsToCreate.map((doc) => {
       const toCreateObject: Partial<typeof doc> = {
         ...doc,
       };
@@ -533,8 +534,7 @@ export const transferDatabaseLocalToRemote = async (
     async () => await localDb.listCollections(fromDbId, [Query.limit(50)])
   );
   const allFromCollections = fromCollections.collections;
-  if (fromCollections.collections.length < 50) {
-  } else {
+  if (fromCollections.collections.length >= 50) {
     lastCollectionId =
       fromCollections.collections[fromCollections.collections.length - 1].$id;
     while (lastCollectionId) {
@@ -555,43 +555,108 @@ export const transferDatabaseLocalToRemote = async (
   }
 
   for (const collection of allFromCollections) {
-    const toCollection = await tryAwaitWithRetry(
+    let toCollection: Models.Collection;
+    const toCollectionExists = await tryAwaitWithRetry(
       async () =>
-        await remoteDb.createCollection(
-          toDbId,
-          collection.$id,
-          collection.name,
-          collection.$permissions,
-          collection.documentSecurity,
-          collection.enabled
-        )
+        await remoteDb.listCollections(toDbId, [
+          Query.equal("$id", collection.$id),
+        ])
     );
-    console.log(`Collection ${toCollection.name} created`);
 
-    for (const attribute of collection.attributes) {
-      await tryAwaitWithRetry(
+    if (toCollectionExists.collections.length > 0) {
+      console.log(`Collection ${collection.name} already exists. Updating...`);
+      toCollection = toCollectionExists.collections[0];
+      // Update collection if needed
+      if (
+        toCollection.name !== collection.name ||
+        toCollection.$permissions !== collection.$permissions ||
+        toCollection.documentSecurity !== collection.documentSecurity ||
+        toCollection.enabled !== collection.enabled
+      ) {
+        toCollection = await tryAwaitWithRetry(
+          async () =>
+            await remoteDb.updateCollection(
+              toDbId,
+              collection.$id,
+              collection.name,
+              collection.$permissions,
+              collection.documentSecurity,
+              collection.enabled
+            )
+        );
+        console.log(`Collection ${toCollection.name} updated`);
+      }
+    } else {
+      toCollection = await tryAwaitWithRetry(
         async () =>
-          await createOrUpdateAttribute(
-            remoteDb,
+          await remoteDb.createCollection(
             toDbId,
-            toCollection,
-            parseAttribute(attribute as any)
+            collection.$id,
+            collection.name,
+            collection.$permissions,
+            collection.documentSecurity,
+            collection.enabled
           )
       );
+      console.log(`Collection ${toCollection.name} created`);
+    }
+
+    // Check and update attributes
+    const existingAttributes = await tryAwaitWithRetry(
+      async () => await remoteDb.listAttributes(toDbId, toCollection.$id)
+    );
+    for (const attribute of collection.attributes) {
+      const parsedAttribute = parseAttribute(attribute as any);
+      const existingAttribute = existingAttributes.attributes.find(
+        // @ts-expect-error
+        (attr) => attr.key === parsedAttribute.key
+      );
+      if (!existingAttribute) {
+        await tryAwaitWithRetry(
+          async () =>
+            await createOrUpdateAttribute(
+              remoteDb,
+              toDbId,
+              toCollection,
+              parsedAttribute
+            )
+        );
+        console.log(`Attribute ${parsedAttribute.key} created`);
+      } else {
+        // Check if attribute needs updating
+        // Note: Appwrite doesn't allow updating most attribute properties
+        // You might need to delete and recreate the attribute if significant changes are needed
+        console.log(`Attribute ${parsedAttribute.key} already exists`);
+      }
     }
 
+    // Check and update indexes
+    const existingIndexes = await tryAwaitWithRetry(
+      async () => await remoteDb.listIndexes(toDbId, toCollection.$id)
+    );
     for (const index of collection.indexes) {
-      await tryAwaitWithRetry(
-        async () =>
-          await remoteDb.createIndex(
-            toDbId,
-            toCollection.$id,
-            index.key,
-            index.type as IndexType,
-            index.attributes,
-            index.orders
-          )
+      const existingIndex = existingIndexes.indexes.find(
+        (idx) => idx.key === index.key
       );
+      if (!existingIndex) {
+        await tryAwaitWithRetry(
+          async () =>
+            await remoteDb.createIndex(
+              toDbId,
+              toCollection.$id,
+              index.key,
+              index.type as IndexType,
+              index.attributes,
+              index.orders
+            )
+        );
+        console.log(`Index ${index.key} created`);
+      } else {
+        // Check if index needs updating
+        // Note: Appwrite doesn't allow updating indexes
+        // You might need to delete and recreate the index if changes are needed
+        console.log(`Index ${index.key} already exists`);
+      }
     }
 
     await transferDocumentsBetweenDbsLocalToRemote(

package/src/utils/helperFunctions.ts

@@ -179,3 +179,6 @@ export const getAppwriteClient = (
     .setProject(projectId)
     .setKey(apiKey);
 };
+
+export const delay = (ms: number) =>
+  new Promise((resolve) => setTimeout(resolve, ms));