appwrite-utils-cli 0.9.51 → 0.9.52

package/README.md

@@ -124,6 +124,7 @@ This updated CLI ensures that developers have robust tools at their fingertips t
 
 ## Changelog
 
+- 0.9.52: Add delay after creating indexes, attributes, and others to prevent `fetch failed` errors during large-scale collection creation
 - 0.9.51: Fix transfer databases, remove "ensure duplicates" check
 - 0.9.5: Fixed not checking for storage bucket for each database (checking the creation status) when importing data
 - 0.9.4: Fixed migrations database ensuring it has the required collections

package/dist/collections/attributes.js

@@ -2,7 +2,7 @@ import { Query } from "node-appwrite";
 import { attributeSchema, parseAttribute, } from "appwrite-utils";
 import { nameToIdMapping, enqueueOperation } from "../migrations/queue.js";
 import _ from "lodash";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 const attributesSame = (databaseAttribute, configAttribute) => {
     return (databaseAttribute.key == configAttribute.key &&
         databaseAttribute.type == configAttribute.type &&
@@ -207,18 +207,18 @@ export const createOrUpdateAttribute = async (db, dbId, collection, attribute) =
 };
 export const createUpdateCollectionAttributes = async (db, dbId, collection, attributes) => {
     console.log(`Creating/Updating attributes for collection: ${collection.name}`);
-    const batchSize = 3; // Size of each batch
+    const batchSize = 3;
     for (let i = 0; i < attributes.length; i += batchSize) {
-        // Slice the attributes array to get a batch of at most batchSize elements
         const batch = attributes.slice(i, i + batchSize);
         const attributePromises = batch.map((attribute) => createOrUpdateAttribute(db, dbId, collection, attribute));
-        // Await the completion of all promises in the current batch
         const results = await Promise.allSettled(attributePromises);
         results.forEach((result) => {
             if (result.status === "rejected") {
                 console.error("An attribute promise was rejected:", result.reason);
             }
         });
+        // Add delay after each batch
+        await delay(2000);
     }
     console.log(`Finished creating/updating attributes for collection: ${collection.name}`);
 };

package/dist/collections/indexes.js

@@ -1,6 +1,6 @@
 import { indexSchema } from "appwrite-utils";
 import { Databases, IndexType, Query } from "node-appwrite";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export const createOrUpdateIndex = async (dbId, db, collectionId, index) => {
     const existingIndex = await db.listIndexes(dbId, collectionId, [
         Query.equal("key", index.key),
@@ -23,5 +23,7 @@ export const createOrUpdateIndex = async (dbId, db, collectionId, index) => {
 export const createOrUpdateIndexes = async (dbId, db, collectionId, indexes) => {
     for (const index of indexes) {
         await tryAwaitWithRetry(async () => await createOrUpdateIndex(dbId, db, collectionId, index));
+        // Add delay after each index creation/update
+        await delay(1000);
     }
 };

package/dist/collections/methods.js

@@ -4,7 +4,7 @@ import { createUpdateCollectionAttributes } from "./attributes.js";
 import { createOrUpdateIndexes } from "./indexes.js";
 import _ from "lodash";
 import { SchemaGenerator } from "../utils/schemaStrings.js";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export const documentExists = async (db, dbId, targetCollectionId, toCreateObject) => {
     // Had to do this because kept running into issues with type checking arrays so, sorry 40ms
     const collection = await db.getCollection(dbId, targetCollectionId);
@@ -109,7 +109,7 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
     if (!configCollections) {
         return;
     }
-    const usedIds = new Set(); // To track IDs used in this operation
+    const usedIds = new Set();
     for (const { attributes, indexes, ...collection } of configCollections) {
         // Prepare permissions for the collection
         const permissions = [];
@@ -149,15 +149,15 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
             let foundColl = deletedCollections?.find((coll) => coll.collectionName.toLowerCase().trim().replace(" ", "") ===
                 collection.name.toLowerCase().trim().replace(" ", ""));
             if (collection.$id) {
-                collectionId = collection.$id; // Always use the provided $id if present
+                collectionId = collection.$id;
             }
             else if (foundColl && !usedIds.has(foundColl.collectionId)) {
-                collectionId = foundColl.collectionId; // Use ID from deleted collection if not already used
+                collectionId = foundColl.collectionId;
             }
             else {
-                collectionId = ID.unique(); // Generate a new unique ID
+                collectionId = ID.unique();
             }
-            usedIds.add(collectionId); // Mark this ID as used
+            usedIds.add(collectionId);
             // Create the collection with the determined ID
             try {
                 collectionToUse = await tryAwaitWithRetry(async () => await database.createCollection(databaseId, collectionId, collection.name, permissions, collection.documentSecurity ?? false, collection.enabled ?? true));
@@ -166,18 +166,24 @@ export const createOrUpdateCollections = async (database, databaseId, config, de
             }
             catch (error) {
                 console.error(`Failed to create collection ${collection.name} with ID ${collectionId}: ${error}`);
-                continue; // Skip to the next collection on failure
+                continue;
             }
         }
         else {
             console.log(`Collection ${collection.name} exists, updating it`);
             await tryAwaitWithRetry(async () => await database.updateCollection(databaseId, collectionToUse.$id, collection.name, permissions, collection.documentSecurity ?? false, collection.enabled ?? true));
         }
+        // Add delay after creating/updating collection
+        await delay(1000);
         // Update attributes and indexes for the collection
         console.log("Creating Attributes");
         await createUpdateCollectionAttributes(database, databaseId, collectionToUse, attributes);
+        // Add delay after creating attributes
+        await delay(1000);
         console.log("Creating Indexes");
         await createOrUpdateIndexes(databaseId, database, collectionToUse.$id, indexes ?? []);
+        // Add delay after creating indexes
+        await delay(1000);
     }
     // Process any remaining tasks in the queue
     await processQueue(database, databaseId);

package/dist/utils/helperFunctions.d.ts

@@ -45,3 +45,4 @@ export declare let numTimesFailedTotal: number;
  */
 export declare const tryAwaitWithRetry: <T>(createFunction: () => Promise<T>, attemptNum?: number, throwError?: boolean) => Promise<T>;
 export declare const getAppwriteClient: (endpoint: string, projectId: string, apiKey: string) => Client;
+export declare const delay: (ms: number) => Promise<unknown>;

package/dist/utils/helperFunctions.js

@@ -116,3 +116,4 @@ export const getAppwriteClient = (endpoint, projectId, apiKey) => {
         .setProject(projectId)
         .setKey(apiKey);
 };
+export const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.9.51",
+  "version": "0.9.52",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/collections/attributes.ts

@@ -6,7 +6,7 @@ import {
 } from "appwrite-utils";
 import { nameToIdMapping, enqueueOperation } from "../migrations/queue.js";
 import _ from "lodash";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 const attributesSame = (
   databaseAttribute: Attribute,
@@ -461,21 +461,22 @@ export const createUpdateCollectionAttributes = async (
     `Creating/Updating attributes for collection: ${collection.name}`
   );
 
-  const batchSize = 3; // Size of each batch
+  const batchSize = 3;
   for (let i = 0; i < attributes.length; i += batchSize) {
-    // Slice the attributes array to get a batch of at most batchSize elements
     const batch = attributes.slice(i, i + batchSize);
     const attributePromises = batch.map((attribute) =>
       createOrUpdateAttribute(db, dbId, collection, attribute)
     );
 
-    // Await the completion of all promises in the current batch
     const results = await Promise.allSettled(attributePromises);
     results.forEach((result) => {
       if (result.status === "rejected") {
         console.error("An attribute promise was rejected:", result.reason);
       }
     });
+
+    // Add delay after each batch
+    await delay(2000);
   }
   console.log(
     `Finished creating/updating attributes for collection: ${collection.name}`

package/src/collections/indexes.ts

@@ -1,6 +1,6 @@
 import { indexSchema, type Index } from "appwrite-utils";
 import { Databases, IndexType, Query, type Models } from "node-appwrite";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 export const createOrUpdateIndex = async (
   dbId: string,
@@ -49,5 +49,7 @@ export const createOrUpdateIndexes = async (
     await tryAwaitWithRetry(
       async () => await createOrUpdateIndex(dbId, db, collectionId, index)
     );
+    // Add delay after each index creation/update
+    await delay(1000);
   }
 };

package/src/collections/methods.ts

@@ -12,7 +12,7 @@ import { createUpdateCollectionAttributes } from "./attributes.js";
 import { createOrUpdateIndexes } from "./indexes.js";
 import _ from "lodash";
 import { SchemaGenerator } from "../utils/schemaStrings.js";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 export const documentExists = async (
   db: Databases,
@@ -163,7 +163,7 @@ export const createOrUpdateCollections = async (
   if (!configCollections) {
     return;
   }
-  const usedIds = new Set(); // To track IDs used in this operation
+  const usedIds = new Set();
 
   for (const { attributes, indexes, ...collection } of configCollections) {
     // Prepare permissions for the collection
@@ -215,14 +215,14 @@ export const createOrUpdateCollections = async (
       );
 
       if (collection.$id) {
-        collectionId = collection.$id; // Always use the provided $id if present
+        collectionId = collection.$id;
       } else if (foundColl && !usedIds.has(foundColl.collectionId)) {
-        collectionId = foundColl.collectionId; // Use ID from deleted collection if not already used
+        collectionId = foundColl.collectionId;
       } else {
-        collectionId = ID.unique(); // Generate a new unique ID
+        collectionId = ID.unique();
       }
 
-      usedIds.add(collectionId); // Mark this ID as used
+      usedIds.add(collectionId);
 
       // Create the collection with the determined ID
       try {
@@ -243,7 +243,7 @@ export const createOrUpdateCollections = async (
         console.error(
           `Failed to create collection ${collection.name} with ID ${collectionId}: ${error}`
         );
-        continue; // Skip to the next collection on failure
+        continue;
       }
     } else {
       console.log(`Collection ${collection.name} exists, updating it`);
@@ -260,6 +260,9 @@ export const createOrUpdateCollections = async (
       );
     }
 
+    // Add delay after creating/updating collection
+    await delay(1000);
+
     // Update attributes and indexes for the collection
     console.log("Creating Attributes");
     await createUpdateCollectionAttributes(
@@ -268,6 +271,10 @@ export const createOrUpdateCollections = async (
       collectionToUse!,
       attributes
     );
+
+    // Add delay after creating attributes
+    await delay(1000);
+
     console.log("Creating Indexes");
     await createOrUpdateIndexes(
       databaseId,
@@ -275,6 +282,9 @@ export const createOrUpdateCollections = async (
       collectionToUse!.$id,
       indexes ?? []
     );
+
+    // Add delay after creating indexes
+    await delay(1000);
   }
   // Process any remaining tasks in the queue
   await processQueue(database, databaseId);

package/src/utils/helperFunctions.ts

@@ -179,3 +179,6 @@ export const getAppwriteClient = (
     .setProject(projectId)
     .setKey(apiKey);
 };
+
+export const delay = (ms: number) =>
+  new Promise((resolve) => setTimeout(resolve, ms));