@njdamstra/appwrite-utils-cli 1.11.2 → 1.11.4

package/dist/cli/commands/migrateCommands.js

@@ -1,5 +1,6 @@
 import inquirer from "inquirer";
 import path from "node:path";
+import fs from "node:fs";
 import { MessageFormatter } from "@njdamstra/appwrite-utils-helpers";
 import { analyzeStringAttributes, executeMigrationPlan, } from "../../migrations/migrateStrings.js";
 export const migrateCommands = {
@@ -94,6 +95,27 @@ export const migrateCommands = {
                 default: path.join(process.cwd(), "migrate-strings-plan.yaml"),
             },
         ]);
+        // Check for existing checkpoint
+        let freshRun = false;
+        const checkpointPath = planPath.replace(/\.ya?ml$/, ".checkpoint.json");
+        if (fs.existsSync(checkpointPath)) {
+            const { cpAction } = await inquirer.prompt([
+                {
+                    type: "list",
+                    name: "cpAction",
+                    message: "Found an existing checkpoint from a previous run. What would you like to do?",
+                    choices: [
+                        { name: "Resume — continue from where it left off", value: "resume" },
+                        { name: "Start fresh — delete checkpoint and start over", value: "fresh" },
+                        { name: "Cancel", value: "cancel" },
+                    ],
+                },
+            ]);
+            if (cpAction === "cancel")
+                return;
+            if (cpAction === "fresh")
+                freshRun = true;
+        }
         const { keepBackups } = await inquirer.prompt([
             {
                 type: "confirm",
@@ -114,6 +136,7 @@ export const migrateCommands = {
             planPath,
             keepBackups,
             dryRun,
+            freshRun,
         };
         try {
             const results = await executeMigrationPlan(controller.adapter, options);

package/dist/main.js

@@ -485,6 +485,11 @@ const argv = yargs(hideBin(process.argv))
     alias: ["migrate-strings-dry-run"],
     type: "boolean",
     description: "Dry run — show what would happen without making changes",
+})
+    .option("migrateStringsFresh", {
+    alias: ["migrate-strings-fresh"],
+    type: "boolean",
+    description: "Ignore existing checkpoint and start migration fresh",
 })
     .parse();
 async function main() {
@@ -674,6 +679,7 @@ async function main() {
                     planPath: argv.migrateStringsExecute,
                     keepBackups: argv.migrateStringsKeepBackups ?? true,
                     dryRun: argv.migrateStringsDryRun ?? false,
+                    freshRun: argv.migrateStringsFresh ?? false,
                 });
                 if (results.failed > 0) {
                     process.exit(1);

package/dist/migrations/migrateStrings.js

@@ -4,6 +4,7 @@ import fs from "node:fs";
 import path from "node:path";
 import inquirer from "inquirer";
 import chalk from "chalk";
+import pLimit from "p-limit";
 import { MessageFormatter, tryAwaitWithRetry, } from "@njdamstra/appwrite-utils-helpers";
 import { ProgressManager } from "../shared/progressManager.js";
 import { MigrationPlanSchema, MigrationCheckpointSchema, suggestTargetType, generateBackupKey, } from "./migrateStringsTypes.js";
@@ -193,6 +194,12 @@ export async function executeMigrationPlan(adapter, options) {
     // Load or create checkpoint
     const checkpointPath = options.checkpointPath ||
         options.planPath.replace(/\.ya?ml$/, ".checkpoint.json");
+    if (options.freshRun && fs.existsSync(checkpointPath)) {
+        fs.unlinkSync(checkpointPath);
+        MessageFormatter.info("Deleted old checkpoint — starting fresh.", {
+            prefix: "Checkpoint",
+        });
+    }
     const checkpoint = loadOrCreateCheckpoint(checkpointPath, options.planPath);
     const batchSize = options.batchSize || 100;
     const batchDelayMs = options.batchDelayMs || 50;
@@ -455,6 +462,7 @@ async function copyAttributeData(adapter, databaseId, collectionId, sourceKey, t
             title: `    Copy ${sourceKey} → ${targetKey}`,
         })
         : undefined;
+    const limit = pLimit(5);
     while (true) {
         const queries = [Query.limit(batchSize)];
         if (lastId)
@@ -463,34 +471,18 @@ async function copyAttributeData(adapter, databaseId, collectionId, sourceKey, t
         const docs = res?.documents || res?.rows || [];
         if (docs.length === 0)
             break;
-        // Batch update: copy sourceKey → targetKey
-        if (adapter.supportsBulkOperations() && adapter.bulkUpsertRows) {
-            const rows = docs
-                .filter((d) => d[sourceKey] !== undefined)
-                .map((d) => ({
-                id: d.$id,
-                data: { [targetKey]: d[sourceKey] },
-            }));
-            if (rows.length > 0) {
-                await tryAwaitWithRetry(() => adapter.bulkUpsertRows({
-                    databaseId,
-                    tableId: collectionId,
-                    rows,
-                }));
-            }
-        }
-        else {
-            for (const doc of docs) {
-                if (doc[sourceKey] === undefined)
-                    continue;
-                await tryAwaitWithRetry(() => adapter.updateRow({
-                    databaseId,
-                    tableId: collectionId,
-                    id: doc.$id,
-                    data: { [targetKey]: doc[sourceKey] },
-                }));
-            }
-        }
+        // Partial update: copy sourceKey → targetKey using updateRow (not bulkUpsert
+        // which requires complete document structure and fails on partial payloads)
+        const updatePromises = docs
+            .filter((d) => d[sourceKey] !== undefined)
+            .map((d) => limit(() => tryAwaitWithRetry(() => adapter.updateRow({
+            databaseId,
+            tableId: collectionId,
+            id: d.$id,
+            data: { [targetKey]: d[sourceKey] },
+        }), 0, true // throwError — surface 400s immediately
+        )));
+        await Promise.all(updatePromises);
         totalCopied += docs.length;
         lastId = docs[docs.length - 1].$id;
         progress?.update(totalCopied);
@@ -513,7 +505,7 @@ async function verifyDataCopy(adapter, databaseId, collectionId, sourceKey, targ
     }));
     const docs = res?.documents || res?.rows || [];
     for (const doc of docs) {
-        if (doc[sourceKey] === undefined)
+        if (!(sourceKey in doc))
             continue;
         if (doc[sourceKey] !== doc[targetKey]) {
             throw new Error(`Verification failed: doc ${doc.$id} has ${sourceKey}=${JSON.stringify(doc[sourceKey])} but ${targetKey}=${JSON.stringify(doc[targetKey])}`);

package/dist/migrations/migrateStringsTypes.d.ts

@@ -191,6 +191,7 @@ export interface ExecuteOptions {
     batchSize?: number;
     batchDelayMs?: number;
     checkpointPath?: string;
+    freshRun?: boolean;
 }
 export declare function suggestTargetType(size: number, hasIndex: boolean): MigrationTargetType;
 export declare function generateBackupKey(originalKey: string): string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@njdamstra/appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "1.11.2",
+  "version": "1.11.4",
   "main": "dist/main.js",
   "type": "module",
   "repository": {

package/src/cli/commands/migrateCommands.ts

@@ -1,5 +1,6 @@
 import inquirer from "inquirer";
 import path from "node:path";
+import fs from "node:fs";
 import { MessageFormatter } from "@njdamstra/appwrite-utils-helpers";
 import type { InteractiveCLI } from "../../interactiveCLI.js";
 import {
@@ -121,6 +122,26 @@ export const migrateCommands = {
       },
     ]);
 
+    // Check for existing checkpoint
+    let freshRun = false;
+    const checkpointPath = planPath.replace(/\.ya?ml$/, ".checkpoint.json");
+    if (fs.existsSync(checkpointPath)) {
+      const { cpAction } = await inquirer.prompt([
+        {
+          type: "list",
+          name: "cpAction",
+          message: "Found an existing checkpoint from a previous run. What would you like to do?",
+          choices: [
+            { name: "Resume — continue from where it left off", value: "resume" },
+            { name: "Start fresh — delete checkpoint and start over", value: "fresh" },
+            { name: "Cancel", value: "cancel" },
+          ],
+        },
+      ]);
+      if (cpAction === "cancel") return;
+      if (cpAction === "fresh") freshRun = true;
+    }
+
     const { keepBackups } = await inquirer.prompt([
       {
         type: "confirm",
@@ -143,6 +164,7 @@ export const migrateCommands = {
       planPath,
       keepBackups,
       dryRun,
+      freshRun,
     };
 
     try {

package/src/main.ts

@@ -89,6 +89,7 @@ interface CliOptions {
   migrateStringsDbIds?: string;
   migrateStringsKeepBackups?: boolean;
   migrateStringsDryRun?: boolean;
+  migrateStringsFresh?: boolean;
 }
 
 type ParsedArgv = ArgumentsCamelCase<CliOptions>;
@@ -659,6 +660,11 @@ const argv = yargs(hideBin(process.argv))
     type: "boolean",
     description: "Dry run — show what would happen without making changes",
   })
+  .option("migrateStringsFresh", {
+    alias: ["migrate-strings-fresh"],
+    type: "boolean",
+    description: "Ignore existing checkpoint and start migration fresh",
+  })
   .parse() as ParsedArgv;
 
 async function main() {
@@ -915,6 +921,7 @@ async function main() {
           planPath: argv.migrateStringsExecute,
           keepBackups: argv.migrateStringsKeepBackups ?? true,
           dryRun: argv.migrateStringsDryRun ?? false,
+          freshRun: argv.migrateStringsFresh ?? false,
         });
         if (results.failed > 0) {
           process.exit(1);

package/src/migrations/migrateStrings.ts

@@ -4,6 +4,7 @@ import fs from "node:fs";
 import path from "node:path";
 import inquirer from "inquirer";
 import chalk from "chalk";
+import pLimit from "p-limit";
 import {
   type DatabaseAdapter,
   MessageFormatter,
@@ -264,6 +265,12 @@ export async function executeMigrationPlan(
   const checkpointPath =
     options.checkpointPath ||
     options.planPath.replace(/\.ya?ml$/, ".checkpoint.json");
+  if (options.freshRun && fs.existsSync(checkpointPath)) {
+    fs.unlinkSync(checkpointPath);
+    MessageFormatter.info("Deleted old checkpoint — starting fresh.", {
+      prefix: "Checkpoint",
+    });
+  }
   const checkpoint = loadOrCreateCheckpoint(checkpointPath, options.planPath);
 
   const batchSize = options.batchSize || 100;
@@ -682,6 +689,8 @@ async function copyAttributeData(
       })
     : undefined;
 
+  const limit = pLimit(5);
+
   while (true) {
     const queries: string[] = [Query.limit(batchSize)];
     if (lastId) queries.push(Query.cursorAfter(lastId));
@@ -693,37 +702,26 @@ async function copyAttributeData(
     const docs = res?.documents || res?.rows || [];
     if (docs.length === 0) break;
 
-    // Batch update: copy sourceKey → targetKey
-    if (adapter.supportsBulkOperations() && adapter.bulkUpsertRows) {
-      const rows = docs
-        .filter((d: any) => d[sourceKey] !== undefined)
-        .map((d: any) => ({
-          id: d.$id,
-          data: { [targetKey]: d[sourceKey] },
-        }));
-
-      if (rows.length > 0) {
-        await tryAwaitWithRetry(() =>
-          adapter.bulkUpsertRows!({
-            databaseId,
-            tableId: collectionId,
-            rows,
-          })
-        );
-      }
-    } else {
-      for (const doc of docs) {
-        if (doc[sourceKey] === undefined) continue;
-        await tryAwaitWithRetry(() =>
-          adapter.updateRow({
-            databaseId,
-            tableId: collectionId,
-            id: doc.$id,
-            data: { [targetKey]: doc[sourceKey] },
-          })
-        );
-      }
-    }
+    // Partial update: copy sourceKey → targetKey using updateRow (not bulkUpsert
+    // which requires complete document structure and fails on partial payloads)
+    const updatePromises = docs
+      .filter((d: any) => d[sourceKey] !== undefined)
+      .map((d: any) =>
+        limit(() =>
+          tryAwaitWithRetry(
+            () =>
+              adapter.updateRow({
+                databaseId,
+                tableId: collectionId,
+                id: d.$id,
+                data: { [targetKey]: d[sourceKey] },
+              }),
+            0,
+            true // throwError — surface 400s immediately
+          )
+        )
+      );
+    await Promise.all(updatePromises);
 
     totalCopied += docs.length;
     lastId = docs[docs.length - 1].$id;
@@ -757,7 +755,7 @@ async function verifyDataCopy(
   );
   const docs = res?.documents || res?.rows || [];
   for (const doc of docs) {
-    if (doc[sourceKey] === undefined) continue;
+    if (!(sourceKey in doc)) continue;
     if (doc[sourceKey] !== doc[targetKey]) {
       throw new Error(
         `Verification failed: doc ${doc.$id} has ${sourceKey}=${JSON.stringify(doc[sourceKey])} but ${targetKey}=${JSON.stringify(doc[targetKey])}`

package/src/migrations/migrateStringsTypes.ts

@@ -118,6 +118,7 @@ export interface ExecuteOptions {
   batchSize?: number;
   batchDelayMs?: number;
   checkpointPath?: string;
+  freshRun?: boolean;
 }
 
 // ── Helper: suggest target type from size + index presence ──