@njdamstra/appwrite-utils-cli 1.11.3 → 1.11.5

package/dist/migrations/migrateStrings.js

@@ -4,6 +4,7 @@ import fs from "node:fs";
 import path from "node:path";
 import inquirer from "inquirer";
 import chalk from "chalk";
+import pLimit from "p-limit";
 import { MessageFormatter, tryAwaitWithRetry, } from "@njdamstra/appwrite-utils-helpers";
 import { ProgressManager } from "../shared/progressManager.js";
 import { MigrationPlanSchema, MigrationCheckpointSchema, suggestTargetType, generateBackupKey, } from "./migrateStringsTypes.js";
@@ -319,7 +320,7 @@ async function migrateOneAttribute(adapter, entry, cpEntry, checkpoint, checkpoi
         MessageFormatter.info(`    Creating backup attribute ${backupKey}...`, {
             prefix: "Migrate",
         });
-        await tryAwaitWithRetry(() => adapter.createAttribute({
+        await createAttributeIfNotExists(adapter, {
             databaseId,
             tableId: collectionId,
             key: backupKey,
@@ -327,7 +328,7 @@ async function migrateOneAttribute(adapter, entry, cpEntry, checkpoint, checkpoi
             size: entry.currentSize,
             required: false, // always optional for backup
             array: entry.isArray,
-        }));
+        });
         const available = await waitForAttribute(adapter, databaseId, collectionId, backupKey);
         if (!available)
             throw new Error(`Backup attribute ${backupKey} stuck`);
@@ -384,7 +385,7 @@ async function migrateOneAttribute(adapter, entry, cpEntry, checkpoint, checkpoi
         if (entry.hasDefault && entry.defaultValue !== undefined) {
             createParams.default = entry.defaultValue;
         }
-        await tryAwaitWithRetry(() => adapter.createAttribute(createParams));
+        await createAttributeIfNotExists(adapter, createParams);
         const available = await waitForAttribute(adapter, databaseId, collectionId, attributeKey);
         if (!available)
             throw new Error(`New attribute ${attributeKey} stuck after creation`);
@@ -461,6 +462,7 @@ async function copyAttributeData(adapter, databaseId, collectionId, sourceKey, t
             title: `    Copy ${sourceKey} → ${targetKey}`,
         })
         : undefined;
+    const limit = pLimit(5);
     while (true) {
         const queries = [Query.limit(batchSize)];
         if (lastId)
@@ -469,34 +471,18 @@ async function copyAttributeData(adapter, databaseId, collectionId, sourceKey, t
         const docs = res?.documents || res?.rows || [];
         if (docs.length === 0)
             break;
-        // Batch update: copy sourceKey → targetKey
-        if (adapter.supportsBulkOperations() && adapter.bulkUpsertRows) {
-            const rows = docs
-                .filter((d) => d[sourceKey] !== undefined)
-                .map((d) => ({
-                id: d.$id,
-                data: { [targetKey]: d[sourceKey] },
-            }));
-            if (rows.length > 0) {
-                await tryAwaitWithRetry(() => adapter.bulkUpsertRows({
-                    databaseId,
-                    tableId: collectionId,
-                    rows,
-                }));
-            }
-        }
-        else {
-            for (const doc of docs) {
-                if (doc[sourceKey] === undefined)
-                    continue;
-                await tryAwaitWithRetry(() => adapter.updateRow({
-                    databaseId,
-                    tableId: collectionId,
-                    id: doc.$id,
-                    data: { [targetKey]: doc[sourceKey] },
-                }));
-            }
-        }
+        // Partial update: copy sourceKey → targetKey using updateRow (not bulkUpsert
+        // which requires complete document structure and fails on partial payloads)
+        const updatePromises = docs
+            .filter((d) => d[sourceKey] !== undefined)
+            .map((d) => limit(() => tryAwaitWithRetry(() => adapter.updateRow({
+            databaseId,
+            tableId: collectionId,
+            id: d.$id,
+            data: { [targetKey]: d[sourceKey] },
+        }), 0, true // throwError — surface 400s immediately
+        )));
+        await Promise.all(updatePromises);
         totalCopied += docs.length;
         lastId = docs[docs.length - 1].$id;
         progress?.update(totalCopied);
@@ -519,9 +505,9 @@ async function verifyDataCopy(adapter, databaseId, collectionId, sourceKey, targ
     }));
     const docs = res?.documents || res?.rows || [];
     for (const doc of docs) {
-        if (doc[sourceKey] === undefined)
+        if (!(sourceKey in doc))
             continue;
-        if (doc[sourceKey] !== doc[targetKey]) {
+        if (JSON.stringify(doc[sourceKey]) !== JSON.stringify(doc[targetKey])) {
             throw new Error(`Verification failed: doc ${doc.$id} has ${sourceKey}=${JSON.stringify(doc[sourceKey])} but ${targetKey}=${JSON.stringify(doc[targetKey])}`);
         }
     }
@@ -529,7 +515,7 @@ async function verifyDataCopy(adapter, databaseId, collectionId, sourceKey, targ
 // ────────────────────────────────────────────────────────
 // Helper: wait for attribute to become available
 // ────────────────────────────────────────────────────────
-async function waitForAttribute(adapter, databaseId, collectionId, key, maxWaitMs = 60_000) {
+async function waitForAttribute(adapter, databaseId, collectionId, key, maxWaitMs = 120_000) {
     const start = Date.now();
     const checkInterval = 2000;
     while (Date.now() - start < maxWaitMs) {
@@ -740,6 +726,22 @@ function printDryRunSummary(plan) {
 // ────────────────────────────────────────────────────────
 // Utility
 // ────────────────────────────────────────────────────────
+async function createAttributeIfNotExists(adapter, params) {
+    try {
+        await tryAwaitWithRetry(() => adapter.createAttribute(params), 0, true);
+    }
+    catch (err) {
+        const code = err?.code || err?.originalError?.code;
+        const type = err?.originalError?.type || "";
+        if (code === 409 || type === "column_already_exists") {
+            MessageFormatter.info(`      (backup attribute already exists, reusing)`, {
+                prefix: "Migrate",
+            });
+            return;
+        }
+        throw err;
+    }
+}
 function delay(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@njdamstra/appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "1.11.3",
+  "version": "1.11.5",
   "main": "dist/main.js",
   "type": "module",
   "repository": {

package/src/migrations/migrateStrings.ts

@@ -4,6 +4,7 @@ import fs from "node:fs";
 import path from "node:path";
 import inquirer from "inquirer";
 import chalk from "chalk";
+import pLimit from "p-limit";
 import {
   type DatabaseAdapter,
   MessageFormatter,
@@ -457,17 +458,15 @@ async function migrateOneAttribute(
     MessageFormatter.info(`    Creating backup attribute ${backupKey}...`, {
       prefix: "Migrate",
     });
-    await tryAwaitWithRetry(() =>
-      adapter.createAttribute({
-        databaseId,
-        tableId: collectionId,
-        key: backupKey,
-        type: "string", // backup keeps original type
-        size: entry.currentSize,
-        required: false, // always optional for backup
-        array: entry.isArray,
-      })
-    );
+    await createAttributeIfNotExists(adapter, {
+      databaseId,
+      tableId: collectionId,
+      key: backupKey,
+      type: "string", // backup keeps original type
+      size: entry.currentSize,
+      required: false, // always optional for backup
+      array: entry.isArray,
+    });
     const available = await waitForAttribute(
       adapter,
       databaseId,
@@ -559,7 +558,7 @@ async function migrateOneAttribute(
       createParams.default = entry.defaultValue;
     }
 
-    await tryAwaitWithRetry(() => adapter.createAttribute(createParams as any));
+    await createAttributeIfNotExists(adapter, createParams as any);
     const available = await waitForAttribute(
       adapter,
       databaseId,
@@ -688,6 +687,8 @@ async function copyAttributeData(
       })
     : undefined;
 
+  const limit = pLimit(5);
+
   while (true) {
     const queries: string[] = [Query.limit(batchSize)];
     if (lastId) queries.push(Query.cursorAfter(lastId));
@@ -699,37 +700,26 @@ async function copyAttributeData(
     const docs = res?.documents || res?.rows || [];
     if (docs.length === 0) break;
 
-    // Batch update: copy sourceKey → targetKey
-    if (adapter.supportsBulkOperations() && adapter.bulkUpsertRows) {
-      const rows = docs
-        .filter((d: any) => d[sourceKey] !== undefined)
-        .map((d: any) => ({
-          id: d.$id,
-          data: { [targetKey]: d[sourceKey] },
-        }));
-
-      if (rows.length > 0) {
-        await tryAwaitWithRetry(() =>
-          adapter.bulkUpsertRows!({
-            databaseId,
-            tableId: collectionId,
-            rows,
-          })
-        );
-      }
-    } else {
-      for (const doc of docs) {
-        if (doc[sourceKey] === undefined) continue;
-        await tryAwaitWithRetry(() =>
-          adapter.updateRow({
-            databaseId,
-            tableId: collectionId,
-            id: doc.$id,
-            data: { [targetKey]: doc[sourceKey] },
-          })
-        );
-      }
-    }
+    // Partial update: copy sourceKey → targetKey using updateRow (not bulkUpsert
+    // which requires complete document structure and fails on partial payloads)
+    const updatePromises = docs
+      .filter((d: any) => d[sourceKey] !== undefined)
+      .map((d: any) =>
+        limit(() =>
+          tryAwaitWithRetry(
+            () =>
+              adapter.updateRow({
+                databaseId,
+                tableId: collectionId,
+                id: d.$id,
+                data: { [targetKey]: d[sourceKey] },
+              }),
+            0,
+            true // throwError — surface 400s immediately
+          )
+        )
+      );
+    await Promise.all(updatePromises);
 
     totalCopied += docs.length;
     lastId = docs[docs.length - 1].$id;
@@ -763,8 +753,8 @@ async function verifyDataCopy(
   );
   const docs = res?.documents || res?.rows || [];
   for (const doc of docs) {
-    if (doc[sourceKey] === undefined) continue;
-    if (doc[sourceKey] !== doc[targetKey]) {
+    if (!(sourceKey in doc)) continue;
+    if (JSON.stringify(doc[sourceKey]) !== JSON.stringify(doc[targetKey])) {
       throw new Error(
         `Verification failed: doc ${doc.$id} has ${sourceKey}=${JSON.stringify(doc[sourceKey])} but ${targetKey}=${JSON.stringify(doc[targetKey])}`
       );
@@ -781,7 +771,7 @@ async function waitForAttribute(
   databaseId: string,
   collectionId: string,
   key: string,
-  maxWaitMs: number = 60_000
+  maxWaitMs: number = 120_000
 ): Promise<boolean> {
   const start = Date.now();
   const checkInterval = 2000;
@@ -1089,6 +1079,25 @@ function printDryRunSummary(plan: MigrationPlan): void {
 // Utility
 // ────────────────────────────────────────────────────────
 
+async function createAttributeIfNotExists(
+  adapter: DatabaseAdapter,
+  params: Parameters<DatabaseAdapter["createAttribute"]>[0]
+): Promise<void> {
+  try {
+    await tryAwaitWithRetry(() => adapter.createAttribute(params), 0, true);
+  } catch (err: any) {
+    const code = err?.code || err?.originalError?.code;
+    const type = err?.originalError?.type || "";
+    if (code === 409 || type === "column_already_exists") {
+      MessageFormatter.info(`      (backup attribute already exists, reusing)`, {
+        prefix: "Migrate",
+      });
+      return;
+    }
+    throw err;
+  }
+}
+
 function delay(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }