@njdamstra/appwrite-utils-cli 1.11.7 → 1.11.9

package/dist/migrations/migrateStrings.js

@@ -7,7 +7,7 @@ import chalk from "chalk";
 import pLimit from "p-limit";
 import { MessageFormatter, tryAwaitWithRetry, } from "@njdamstra/appwrite-utils-helpers";
 import { ProgressManager } from "../shared/progressManager.js";
-import { MigrationPlanSchema, MigrationCheckpointSchema, suggestTargetType, generateBackupKey, } from "./migrateStringsTypes.js";
+import { MigrationPlanSchema, MigrationCheckpointSchema, suggestTargetType, generateBackupKey, generateArchiveKey, } from "./migrateStringsTypes.js";
 // ────────────────────────────────────────────────────────
 // Phase 1: Analyze — queries Appwrite server for real state
 // ────────────────────────────────────────────────────────
@@ -201,8 +201,20 @@ export async function executeMigrationPlan(adapter, options) {
         });
     }
     const checkpoint = loadOrCreateCheckpoint(checkpointPath, options.planPath);
-    const batchSize = options.batchSize || 100;
-    const batchDelayMs = options.batchDelayMs || 50;
+    // Detect old-version checkpoints that used 2-copy flow
+    if (!checkpoint.version || checkpoint.version < 2) {
+        const hasProgress = checkpoint.entries.some((e) => e.phase !== "pending" &&
+            e.phase !== "completed" &&
+            e.phase !== "failed");
+        if (hasProgress) {
+            MessageFormatter.warning("Checkpoint from older migration version detected with in-progress entries. " +
+                "Use --fresh to restart with optimized rename-based flow.", { prefix: "Checkpoint" });
+        }
+        checkpoint.version = 2;
+        saveCheckpoint(checkpoint, checkpointPath);
+    }
+    const batchSize = options.batchSize || 500;
+    const batchDelayMs = options.batchDelayMs || 0;
     // Group by database/collection
     const groups = new Map();
     for (const entry of migrateEntries) {
@@ -250,6 +262,31 @@ export async function executeMigrationPlan(adapter, options) {
             skipped += entries.length;
             continue;
         }
+        // Temporarily set ALL required attributes in this collection to optional.
+        // Appwrite rejects partial updateRow calls if a required attribute is missing
+        // from the payload — even if the field exists in the document. This affects
+        // every data-copy step during migration.
+        const schemaRes = await tryAwaitWithRetry(() => adapter.getTable({ databaseId: first.databaseId, tableId: first.collectionId }));
+        const allAttrs = schemaRes?.data?.attributes || schemaRes?.data?.columns || [];
+        const originallyRequired = allAttrs
+            .filter((a) => a.required === true && a.status === "available")
+            .map((a) => a.key);
+        if (originallyRequired.length > 0) {
+            MessageFormatter.info(`  Temporarily setting ${originallyRequired.length} required attribute(s) to optional...`, { prefix: "Execute" });
+            for (const key of originallyRequired) {
+                try {
+                    await tryAwaitWithRetry(() => adapter.updateAttribute({
+                        databaseId: first.databaseId,
+                        tableId: first.collectionId,
+                        key,
+                        required: false,
+                    }));
+                }
+                catch {
+                    // Non-fatal — attribute might not support updating required
+                }
+            }
+        }
         // Migrate each attribute in this collection
         for (const entry of entries) {
             const cpEntry = getOrCreateCheckpointEntry(checkpoint, entry);
@@ -275,25 +312,23 @@ export async function executeMigrationPlan(adapter, options) {
                 MessageFormatter.error(`  ${entry.attributeKey}: FAILED — ${cpEntry.error}`, undefined, { prefix: "Execute" });
             }
         }
-        // Restore required flags after all attributes in this collection are done.
-        // This must happen AFTER all migrations to avoid partial-update validation
-        // errors (Appwrite rejects updateRow if a required attribute is missing
-        // from the payload, even for partial updates).
-        const completedRequired = entries.filter((e) => {
-            const cp = findCheckpointEntry(checkpoint, e);
-            return cp?.phase === "completed" && e.isRequired;
-        });
-        for (const entry of completedRequired) {
-            try {
-                await tryAwaitWithRetry(() => adapter.updateAttribute({
-                    databaseId: entry.databaseId,
-                    tableId: entry.collectionId,
-                    key: entry.attributeKey,
-                    required: true,
-                }));
-            }
-            catch {
-                MessageFormatter.info(`  Warning: could not set ${entry.attributeKey} back to required`, { prefix: "Execute" });
+        // Restore required flags for ALL originally-required attributes.
+        // This covers both migrated attributes (recreated as optional) and
+        // non-migrated attributes (temporarily set to optional above).
+        if (originallyRequired.length > 0) {
+            MessageFormatter.info(`  Restoring ${originallyRequired.length} required attribute(s)...`, { prefix: "Execute" });
+            for (const key of originallyRequired) {
+                try {
+                    await tryAwaitWithRetry(() => adapter.updateAttribute({
+                        databaseId: first.databaseId,
+                        tableId: first.collectionId,
+                        key,
+                        required: true,
+                    }));
+                }
+                catch {
+                    MessageFormatter.info(`  Warning: could not restore required flag for ${key}`, { prefix: "Execute" });
+                }
             }
         }
         // After collection completes, offer to update local YAML
@@ -336,26 +371,27 @@ async function migrateOneAttribute(adapter, entry, cpEntry, checkpoint, checkpoi
         checkpoint.lastUpdatedAt = new Date().toISOString();
         saveCheckpoint(checkpoint, checkpointPath);
     };
-    // Step 1: Create backup attribute
+    // Step 1: Create backup attribute as TARGET type directly
     if (phaseIndex(cpEntry.phase) < phaseIndex("backup_created")) {
-        MessageFormatter.info(`    Creating backup attribute ${backupKey}...`, {
-            prefix: "Migrate",
-        });
-        await createAttributeIfNotExists(adapter, {
+        MessageFormatter.info(`    Creating backup attribute ${backupKey} as ${targetType}...`, { prefix: "Migrate" });
+        const createParams = {
             databaseId,
             tableId: collectionId,
             key: backupKey,
-            type: "string", // backup keeps original type
-            size: entry.currentSize,
-            required: false, // always optional for backup
+            type: targetType,
+            required: false,
             array: entry.isArray,
-        });
+        };
+        if (targetType === "varchar" && targetSize) {
+            createParams.size = targetSize;
+        }
+        await createAttributeIfNotExists(adapter, createParams);
         const available = await waitForAttribute(adapter, databaseId, collectionId, backupKey);
         if (!available)
             throw new Error(`Backup attribute ${backupKey} stuck`);
         advance("backup_created");
     }
-    // Step 2: Copy data to backup
+    // Step 2: Copy data to backup (only data copy needed)
     if (phaseIndex(cpEntry.phase) < phaseIndex("data_copied_to_backup")) {
         MessageFormatter.info(`    Copying data to backup ${backupKey}...`, {
             prefix: "Migrate",
@@ -368,85 +404,64 @@ async function migrateOneAttribute(adapter, entry, cpEntry, checkpoint, checkpoi
         await verifyDataCopy(adapter, databaseId, collectionId, attributeKey, backupKey);
         advance("data_verified_backup");
     }
-    // Step 4: Delete indexes + original attribute
-    if (phaseIndex(cpEntry.phase) < phaseIndex("original_deleted")) {
-        // Save and delete affected indexes
+    // Step 4: Clear original attribute name (delete or archive)
+    if (phaseIndex(cpEntry.phase) < phaseIndex("original_cleared")) {
+        // Delete indexes first (required before delete or rename)
         if (entry.indexesAffected.length > 0) {
-            MessageFormatter.info(`    Removing ${entry.indexesAffected.length} affected index(es)...`, {
-                prefix: "Migrate",
-            });
+            MessageFormatter.info(`    Removing ${entry.indexesAffected.length} affected index(es)...`, { prefix: "Migrate" });
             await saveAndDeleteIndexes(adapter, databaseId, collectionId, entry.indexesAffected, cpEntry);
             saveCheckpoint(checkpoint, checkpointPath);
         }
-        MessageFormatter.info(`    Deleting original attribute ${attributeKey}...`, {
-            prefix: "Migrate",
-        });
-        await tryAwaitWithRetry(() => adapter.deleteAttribute({
-            databaseId,
-            tableId: collectionId,
-            key: attributeKey,
-        }));
-        await waitForAttributeGone(adapter, databaseId, collectionId, attributeKey);
-        advance("original_deleted");
-    }
-    // Step 5: Create new attribute with target type
-    if (phaseIndex(cpEntry.phase) < phaseIndex("new_attr_created")) {
-        MessageFormatter.info(`    Creating new attribute ${attributeKey} as ${targetType}...`, { prefix: "Migrate" });
-        const createParams = {
-            databaseId,
-            tableId: collectionId,
-            key: attributeKey,
-            type: targetType,
-            required: false, // create as optional first — data needs to be copied back
-            array: entry.isArray,
-        };
-        if (targetType === "varchar" && targetSize) {
-            createParams.size = targetSize;
+        if (opts.keepBackups) {
+            // ARCHIVE: rename original → og_X (preserves original data under new name)
+            const archiveKey = generateArchiveKey(attributeKey);
+            MessageFormatter.info(`    Archiving original ${attributeKey} → ${archiveKey}...`, { prefix: "Migrate" });
+            await tryAwaitWithRetry(() => adapter.updateAttribute({
+                databaseId,
+                tableId: collectionId,
+                key: attributeKey,
+                newKey: archiveKey,
+                required: false,
+            }));
+            await waitForAttribute(adapter, databaseId, collectionId, archiveKey);
         }
-        if (entry.hasDefault && entry.defaultValue !== undefined) {
-            createParams.default = entry.defaultValue;
+        else {
+            // DELETE: remove original attribute entirely
+            MessageFormatter.info(`    Deleting original attribute ${attributeKey}...`, { prefix: "Migrate" });
+            await tryAwaitWithRetry(() => adapter.deleteAttribute({
+                databaseId,
+                tableId: collectionId,
+                key: attributeKey,
+            }));
+            await waitForAttributeGone(adapter, databaseId, collectionId, attributeKey);
         }
-        await createAttributeIfNotExists(adapter, createParams);
-        const available = await waitForAttribute(adapter, databaseId, collectionId, attributeKey);
-        if (!available)
-            throw new Error(`New attribute ${attributeKey} stuck after creation`);
-        advance("new_attr_created");
-    }
-    // Step 6: Copy data back from backup
-    if (phaseIndex(cpEntry.phase) < phaseIndex("data_copied_back")) {
-        MessageFormatter.info(`    Copying data back from backup...`, {
-            prefix: "Migrate",
-        });
-        await copyAttributeData(adapter, databaseId, collectionId, backupKey, attributeKey, opts.batchSize, opts.batchDelayMs);
-        advance("data_copied_back");
+        advance("original_cleared");
     }
-    // Step 7: Verify final data
-    if (phaseIndex(cpEntry.phase) < phaseIndex("data_verified_final")) {
-        await verifyDataCopy(adapter, databaseId, collectionId, backupKey, attributeKey);
-        advance("data_verified_final");
+    // Step 5: Rename backup to original name
+    if (phaseIndex(cpEntry.phase) < phaseIndex("backup_renamed")) {
+        MessageFormatter.info(`    Renaming ${backupKey} → ${attributeKey}...`, { prefix: "Migrate" });
+        await tryAwaitWithRetry(() => adapter.updateAttribute({
+            databaseId,
+            tableId: collectionId,
+            key: backupKey,
+            newKey: attributeKey,
+            required: false,
+            ...(entry.hasDefault && entry.defaultValue !== undefined
+                ? { default: entry.defaultValue }
+                : {}),
+        }));
+        await waitForAttribute(adapter, databaseId, collectionId, attributeKey);
+        advance("backup_renamed");
     }
-    // Step 8: Recreate indexes + delete backup
-    if (phaseIndex(cpEntry.phase) < phaseIndex("backup_deleted")) {
-        // Recreate indexes
+    // Step 6: Recreate indexes
+    if (phaseIndex(cpEntry.phase) < phaseIndex("indexes_recreated")) {
         if (cpEntry.storedIndexes.length > 0) {
             MessageFormatter.info(`    Recreating ${cpEntry.storedIndexes.length} index(es)...`, { prefix: "Migrate" });
             await recreateIndexes(adapter, databaseId, collectionId, cpEntry);
         }
-        // Delete backup (unless keepBackups)
-        if (!opts.keepBackups) {
-            MessageFormatter.info(`    Deleting backup attribute ${backupKey}...`, {
-                prefix: "Migrate",
-            });
-            await tryAwaitWithRetry(() => adapter.deleteAttribute({
-                databaseId,
-                tableId: collectionId,
-                key: backupKey,
-            }));
-            await waitForAttributeGone(adapter, databaseId, collectionId, backupKey);
-        }
-        advance("backup_deleted");
+        advance("indexes_recreated");
     }
-    // Step 9: Mark completed
+    // Mark completed
     // NOTE: required flag is restored AFTER all attributes in the collection
     // are migrated, to avoid partial-update validation errors on other attributes.
     advance("completed");
@@ -470,7 +485,7 @@ async function copyAttributeData(adapter, databaseId, collectionId, sourceKey, t
             title: `    Copy ${sourceKey} → ${targetKey}`,
         })
         : undefined;
-    const limit = pLimit(5);
+    const limit = pLimit(25);
     while (true) {
         const queries = [Query.limit(batchSize)];
         if (lastId)
@@ -493,6 +508,15 @@ async function copyAttributeData(adapter, databaseId, collectionId, sourceKey, t
         await Promise.all(updatePromises);
         totalCopied += docs.length;
         lastId = docs[docs.length - 1].$id;
+        // Appwrite caps result.total at 5000 — adjust progress bar if we exceed it
+        if (progress && totalDocs && totalCopied > totalDocs) {
+            // Estimate remaining: if we haven't hit the last page, assume at least one more batch
+            const estimatedTotal = docs.length < batchSize
+                ? totalCopied
+                : totalCopied + batchSize;
+            progress.setTotal(estimatedTotal);
+            totalDocs = estimatedTotal;
+        }
         progress?.update(totalCopied);
         if (docs.length < batchSize)
             break; // last page
@@ -645,6 +669,7 @@ function loadOrCreateCheckpoint(checkpointPath, planFile) {
     }
     const now = new Date().toISOString();
     return {
+        version: 2,
         planFile,
         startedAt: now,
         lastUpdatedAt: now,
@@ -685,11 +710,9 @@ const PHASE_ORDER = [
     "backup_created",
     "data_copied_to_backup",
     "data_verified_backup",
-    "original_deleted",
-    "new_attr_created",
-    "data_copied_back",
-    "data_verified_final",
-    "backup_deleted",
+    "original_cleared",
+    "backup_renamed",
+    "indexes_recreated",
     "completed",
 ];
 function phaseIndex(phase) {
@@ -701,8 +724,9 @@ function phaseIndex(phase) {
 // ────────────────────────────────────────────────────────
 function printDryRunSummary(plan) {
     console.log("");
-    console.log(chalk.bold("Dry Run — What Would Happen:"));
+    console.log(chalk.bold("Dry Run — What Would Happen (rename-based flow):"));
     console.log(chalk.gray("─".repeat(50)));
+    console.log(chalk.dim("  Flow: create mig_X (target type) → copy data → delete/archive original → rename mig_X → X"));
     const groups = new Map();
     for (const entry of plan.entries) {
         if (entry.action !== "migrate")

package/dist/migrations/migrateStringsTypes.d.ts

@@ -99,6 +99,9 @@ export declare const CheckpointPhase: z.ZodEnum<{
     backup_created: "backup_created";
     data_copied_to_backup: "data_copied_to_backup";
     data_verified_backup: "data_verified_backup";
+    original_cleared: "original_cleared";
+    backup_renamed: "backup_renamed";
+    indexes_recreated: "indexes_recreated";
     original_deleted: "original_deleted";
     new_attr_created: "new_attr_created";
     data_copied_back: "data_copied_back";
@@ -118,6 +121,9 @@ export declare const CheckpointEntrySchema: z.ZodObject<{
         backup_created: "backup_created";
         data_copied_to_backup: "data_copied_to_backup";
         data_verified_backup: "data_verified_backup";
+        original_cleared: "original_cleared";
+        backup_renamed: "backup_renamed";
+        indexes_recreated: "indexes_recreated";
         original_deleted: "original_deleted";
         new_attr_created: "new_attr_created";
         data_copied_back: "data_copied_back";
@@ -141,6 +147,7 @@ export declare const CheckpointEntrySchema: z.ZodObject<{
 }, z.core.$strip>;
 export type CheckpointEntry = z.infer<typeof CheckpointEntrySchema>;
 export declare const MigrationCheckpointSchema: z.ZodObject<{
+    version: z.ZodDefault<z.ZodNumber>;
     planFile: z.ZodString;
     startedAt: z.ZodString;
     lastUpdatedAt: z.ZodString;
@@ -156,6 +163,9 @@ export declare const MigrationCheckpointSchema: z.ZodObject<{
             backup_created: "backup_created";
             data_copied_to_backup: "data_copied_to_backup";
             data_verified_backup: "data_verified_backup";
+            original_cleared: "original_cleared";
+            backup_renamed: "backup_renamed";
+            indexes_recreated: "indexes_recreated";
             original_deleted: "original_deleted";
             new_attr_created: "new_attr_created";
             data_copied_back: "data_copied_back";
@@ -195,3 +205,4 @@ export interface ExecuteOptions {
 }
 export declare function suggestTargetType(size: number, hasIndex: boolean): MigrationTargetType;
 export declare function generateBackupKey(originalKey: string): string;
+export declare function generateArchiveKey(originalKey: string): string;

package/dist/migrations/migrateStringsTypes.js

@@ -49,13 +49,17 @@ export const CheckpointPhase = z.enum([
     "backup_created",
     "data_copied_to_backup",
     "data_verified_backup",
+    "original_cleared",
+    "backup_renamed",
+    "indexes_recreated",
+    "completed",
+    "failed",
+    // Legacy phases (kept for Zod validation of old v1 checkpoints)
     "original_deleted",
     "new_attr_created",
     "data_copied_back",
     "data_verified_final",
     "backup_deleted",
-    "completed",
-    "failed",
 ]);
 export const CheckpointEntrySchema = z.object({
     databaseId: z.string(),
@@ -77,6 +81,7 @@ export const CheckpointEntrySchema = z.object({
         .default([]),
 });
 export const MigrationCheckpointSchema = z.object({
+    version: z.number().default(1),
     planFile: z.string(),
     startedAt: z.string(),
     lastUpdatedAt: z.string(),
@@ -109,6 +114,17 @@ export function generateBackupKey(originalKey) {
     const maxOrigLen = MAX_KEY_LENGTH - TRUNC_PREFIX.length - 1 - 4;
     return `${TRUNC_PREFIX}${originalKey.slice(0, maxOrigLen)}_${hash}`;
 }
+const ARCHIVE_PREFIX = "og_";
+export function generateArchiveKey(originalKey) {
+    const candidate = `${ARCHIVE_PREFIX}${originalKey}`;
+    if (candidate.length <= MAX_KEY_LENGTH) {
+        return candidate;
+    }
+    const hash = simpleHash(originalKey);
+    const TRUNC_PREFIX = "o_";
+    const maxOrigLen = MAX_KEY_LENGTH - TRUNC_PREFIX.length - 1 - 4;
+    return `${TRUNC_PREFIX}${originalKey.slice(0, maxOrigLen)}_${hash}`;
+}
 function simpleHash(str) {
     let h = 0;
     for (let i = 0; i < str.length; i++) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@njdamstra/appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "1.11.7",
+  "version": "1.11.9",
   "main": "dist/main.js",
   "type": "module",
   "repository": {
@@ -40,7 +40,7 @@
     "@types/json-schema": "^7.0.15",
     "@types/yargs": "^17.0.33",
     "@njdamstra/appwrite-utils": "^1.7.1",
-    "@njdamstra/appwrite-utils-helpers": "^0.1.2",
+    "@njdamstra/appwrite-utils-helpers": "^0.1.3",
     "chalk": "^5.4.1",
     "cli-progress": "^3.12.0",
     "commander": "^12.1.0",

package/src/migrations/migrateStrings.ts

@@ -24,6 +24,7 @@ import {
   MigrationCheckpointSchema,
   suggestTargetType,
   generateBackupKey,
+  generateArchiveKey,
 } from "./migrateStringsTypes.js";
 
 // ────────────────────────────────────────────────────────
@@ -273,8 +274,27 @@ export async function executeMigrationPlan(
   }
   const checkpoint = loadOrCreateCheckpoint(checkpointPath, options.planPath);
 
-  const batchSize = options.batchSize || 100;
-  const batchDelayMs = options.batchDelayMs || 50;
+  // Detect old-version checkpoints that used 2-copy flow
+  if (!checkpoint.version || checkpoint.version < 2) {
+    const hasProgress = checkpoint.entries.some(
+      (e) =>
+        e.phase !== "pending" &&
+        e.phase !== "completed" &&
+        e.phase !== "failed"
+    );
+    if (hasProgress) {
+      MessageFormatter.warning(
+        "Checkpoint from older migration version detected with in-progress entries. " +
+          "Use --fresh to restart with optimized rename-based flow.",
+        { prefix: "Checkpoint" }
+      );
+    }
+    checkpoint.version = 2;
+    saveCheckpoint(checkpoint, checkpointPath);
+  }
+
+  const batchSize = options.batchSize || 500;
+  const batchDelayMs = options.batchDelayMs || 0;
 
   // Group by database/collection
   const groups = new Map<string, MigrationPlanEntry[]>();
@@ -338,6 +358,40 @@ export async function executeMigrationPlan(
       continue;
     }
 
+    // Temporarily set ALL required attributes in this collection to optional.
+    // Appwrite rejects partial updateRow calls if a required attribute is missing
+    // from the payload — even if the field exists in the document. This affects
+    // every data-copy step during migration.
+    const schemaRes = await tryAwaitWithRetry(() =>
+      adapter.getTable({ databaseId: first.databaseId, tableId: first.collectionId })
+    );
+    const allAttrs: any[] =
+      schemaRes?.data?.attributes || schemaRes?.data?.columns || [];
+    const originallyRequired = allAttrs
+      .filter((a: any) => a.required === true && a.status === "available")
+      .map((a: any) => a.key as string);
+
+    if (originallyRequired.length > 0) {
+      MessageFormatter.info(
+        `  Temporarily setting ${originallyRequired.length} required attribute(s) to optional...`,
+        { prefix: "Execute" }
+      );
+      for (const key of originallyRequired) {
+        try {
+          await tryAwaitWithRetry(() =>
+            adapter.updateAttribute({
+              databaseId: first.databaseId,
+              tableId: first.collectionId,
+              key,
+              required: false,
+            } as any)
+          );
+        } catch {
+          // Non-fatal — attribute might not support updating required
+        }
+      }
+    }
+
     // Migrate each attribute in this collection
     for (const entry of entries) {
       const cpEntry = getOrCreateCheckpointEntry(checkpoint, entry);
@@ -382,29 +436,30 @@ export async function executeMigrationPlan(
       }
     }
 
-    // Restore required flags after all attributes in this collection are done.
-    // This must happen AFTER all migrations to avoid partial-update validation
-    // errors (Appwrite rejects updateRow if a required attribute is missing
-    // from the payload, even for partial updates).
-    const completedRequired = entries.filter((e) => {
-      const cp = findCheckpointEntry(checkpoint, e);
-      return cp?.phase === "completed" && e.isRequired;
-    });
-    for (const entry of completedRequired) {
-      try {
-        await tryAwaitWithRetry(() =>
-          adapter.updateAttribute({
-            databaseId: entry.databaseId,
-            tableId: entry.collectionId,
-            key: entry.attributeKey,
-            required: true,
-          } as any)
-        );
-      } catch {
-        MessageFormatter.info(
-          `  Warning: could not set ${entry.attributeKey} back to required`,
-          { prefix: "Execute" }
-        );
+    // Restore required flags for ALL originally-required attributes.
+    // This covers both migrated attributes (recreated as optional) and
+    // non-migrated attributes (temporarily set to optional above).
+    if (originallyRequired.length > 0) {
+      MessageFormatter.info(
+        `  Restoring ${originallyRequired.length} required attribute(s)...`,
+        { prefix: "Execute" }
+      );
+      for (const key of originallyRequired) {
+        try {
+          await tryAwaitWithRetry(() =>
+            adapter.updateAttribute({
+              databaseId: first.databaseId,
+              tableId: first.collectionId,
+              key,
+              required: true,
+            } as any)
+          );
+        } catch {
+          MessageFormatter.info(
+            `  Warning: could not restore required flag for ${key}`,
+            { prefix: "Execute" }
+          );
+        }
       }
     }
 
@@ -449,7 +504,7 @@ export async function executeMigrationPlan(
 }
 
 // ────────────────────────────────────────────────────────
-// Single attribute migration (9 phases)
+// Single attribute migration (rename-based, 6 steps)
 // ────────────────────────────────────────────────────────
 
 interface MigrateOneOptions {
@@ -476,20 +531,24 @@ async function migrateOneAttribute(
     saveCheckpoint(checkpoint, checkpointPath);
   };
 
-  // Step 1: Create backup attribute
+  // Step 1: Create backup attribute as TARGET type directly
   if (phaseIndex(cpEntry.phase) < phaseIndex("backup_created")) {
-    MessageFormatter.info(`    Creating backup attribute ${backupKey}...`, {
-      prefix: "Migrate",
-    });
-    await createAttributeIfNotExists(adapter, {
+    MessageFormatter.info(
+      `    Creating backup attribute ${backupKey} as ${targetType}...`,
+      { prefix: "Migrate" }
+    );
+    const createParams: Record<string, any> = {
       databaseId,
       tableId: collectionId,
       key: backupKey,
-      type: "string", // backup keeps original type
-      size: entry.currentSize,
-      required: false, // always optional for backup
+      type: targetType,
+      required: false,
       array: entry.isArray,
-    });
+    };
+    if (targetType === "varchar" && targetSize) {
+      createParams.size = targetSize;
+    }
+    await createAttributeIfNotExists(adapter, createParams as any);
     const available = await waitForAttribute(
       adapter,
       databaseId,
@@ -500,7 +559,7 @@ async function migrateOneAttribute(
     advance("backup_created");
   }
 
-  // Step 2: Copy data to backup
+  // Step 2: Copy data to backup (only data copy needed)
   if (phaseIndex(cpEntry.phase) < phaseIndex("data_copied_to_backup")) {
     MessageFormatter.info(`    Copying data to backup ${backupKey}...`, {
       prefix: "Migrate",
@@ -529,13 +588,14 @@ async function migrateOneAttribute(
     advance("data_verified_backup");
   }
 
-  // Step 4: Delete indexes + original attribute
-  if (phaseIndex(cpEntry.phase) < phaseIndex("original_deleted")) {
-    // Save and delete affected indexes
+  // Step 4: Clear original attribute name (delete or archive)
+  if (phaseIndex(cpEntry.phase) < phaseIndex("original_cleared")) {
+    // Delete indexes first (required before delete or rename)
     if (entry.indexesAffected.length > 0) {
-      MessageFormatter.info(`    Removing ${entry.indexesAffected.length} affected index(es)...`, {
-        prefix: "Migrate",
-      });
+      MessageFormatter.info(
+        `    Removing ${entry.indexesAffected.length} affected index(es)...`,
+        { prefix: "Migrate" }
+      );
       await saveAndDeleteIndexes(
         adapter,
         databaseId,
@@ -546,85 +606,70 @@ async function migrateOneAttribute(
       saveCheckpoint(checkpoint, checkpointPath);
     }
 
-    MessageFormatter.info(`    Deleting original attribute ${attributeKey}...`, {
-      prefix: "Migrate",
-    });
-    await tryAwaitWithRetry(() =>
-      adapter.deleteAttribute({
+    if (opts.keepBackups) {
+      // ARCHIVE: rename original → og_X (preserves original data under new name)
+      const archiveKey = generateArchiveKey(attributeKey);
+      MessageFormatter.info(
+        `    Archiving original ${attributeKey} → ${archiveKey}...`,
+        { prefix: "Migrate" }
+      );
+      await tryAwaitWithRetry(() =>
+        adapter.updateAttribute({
+          databaseId,
+          tableId: collectionId,
+          key: attributeKey,
+          newKey: archiveKey,
+          required: false,
+        })
+      );
+      await waitForAttribute(adapter, databaseId, collectionId, archiveKey);
+    } else {
+      // DELETE: remove original attribute entirely
+      MessageFormatter.info(
+        `    Deleting original attribute ${attributeKey}...`,
+        { prefix: "Migrate" }
+      );
+      await tryAwaitWithRetry(() =>
+        adapter.deleteAttribute({
+          databaseId,
+          tableId: collectionId,
+          key: attributeKey,
+        })
+      );
+      await waitForAttributeGone(
+        adapter,
         databaseId,
-        tableId: collectionId,
-        key: attributeKey,
-      })
-    );
-    await waitForAttributeGone(adapter, databaseId, collectionId, attributeKey);
-    advance("original_deleted");
+        collectionId,
+        attributeKey
+      );
+    }
+    advance("original_cleared");
   }
 
-  // Step 5: Create new attribute with target type
-  if (phaseIndex(cpEntry.phase) < phaseIndex("new_attr_created")) {
+  // Step 5: Rename backup to original name
+  if (phaseIndex(cpEntry.phase) < phaseIndex("backup_renamed")) {
     MessageFormatter.info(
-      `    Creating new attribute ${attributeKey} as ${targetType}...`,
+      `    Renaming ${backupKey} → ${attributeKey}...`,
       { prefix: "Migrate" }
     );
-    const createParams: Record<string, any> = {
-      databaseId,
-      tableId: collectionId,
-      key: attributeKey,
-      type: targetType,
-      required: false, // create as optional first — data needs to be copied back
-      array: entry.isArray,
-    };
-    if (targetType === "varchar" && targetSize) {
-      createParams.size = targetSize;
-    }
-    if (entry.hasDefault && entry.defaultValue !== undefined) {
-      createParams.default = entry.defaultValue;
-    }
-
-    await createAttributeIfNotExists(adapter, createParams as any);
-    const available = await waitForAttribute(
-      adapter,
-      databaseId,
-      collectionId,
-      attributeKey
-    );
-    if (!available)
-      throw new Error(`New attribute ${attributeKey} stuck after creation`);
-    advance("new_attr_created");
-  }
-
-  // Step 6: Copy data back from backup
-  if (phaseIndex(cpEntry.phase) < phaseIndex("data_copied_back")) {
-    MessageFormatter.info(`    Copying data back from backup...`, {
-      prefix: "Migrate",
-    });
-    await copyAttributeData(
-      adapter,
-      databaseId,
-      collectionId,
-      backupKey,
-      attributeKey,
-      opts.batchSize,
-      opts.batchDelayMs
-    );
-    advance("data_copied_back");
-  }
-
-  // Step 7: Verify final data
-  if (phaseIndex(cpEntry.phase) < phaseIndex("data_verified_final")) {
-    await verifyDataCopy(
-      adapter,
-      databaseId,
-      collectionId,
-      backupKey,
-      attributeKey
+    await tryAwaitWithRetry(() =>
+      adapter.updateAttribute({
+        databaseId,
+        tableId: collectionId,
+        key: backupKey,
+        newKey: attributeKey,
+        required: false,
+        ...(entry.hasDefault && entry.defaultValue !== undefined
+          ? { default: entry.defaultValue }
+          : {}),
+      })
     );
-    advance("data_verified_final");
+    await waitForAttribute(adapter, databaseId, collectionId, attributeKey);
+    advance("backup_renamed");
   }
 
-  // Step 8: Recreate indexes + delete backup
-  if (phaseIndex(cpEntry.phase) < phaseIndex("backup_deleted")) {
-    // Recreate indexes
+  // Step 6: Recreate indexes
+  if (phaseIndex(cpEntry.phase) < phaseIndex("indexes_recreated")) {
     if (cpEntry.storedIndexes.length > 0) {
       MessageFormatter.info(
         `    Recreating ${cpEntry.storedIndexes.length} index(es)...`,
@@ -632,30 +677,10 @@ async function migrateOneAttribute(
       );
       await recreateIndexes(adapter, databaseId, collectionId, cpEntry);
     }
-
-    // Delete backup (unless keepBackups)
-    if (!opts.keepBackups) {
-      MessageFormatter.info(`    Deleting backup attribute ${backupKey}...`, {
-        prefix: "Migrate",
-      });
-      await tryAwaitWithRetry(() =>
-        adapter.deleteAttribute({
-          databaseId,
-          tableId: collectionId,
-          key: backupKey,
-        })
-      );
-      await waitForAttributeGone(
-        adapter,
-        databaseId,
-        collectionId,
-        backupKey
-      );
-    }
-    advance("backup_deleted");
+    advance("indexes_recreated");
   }
 
-  // Step 9: Mark completed
+  // Mark completed
   // NOTE: required flag is restored AFTER all attributes in the collection
   // are migrated, to avoid partial-update validation errors on other attributes.
   advance("completed");
@@ -693,7 +718,7 @@ async function copyAttributeData(
       })
     : undefined;
 
-  const limit = pLimit(5);
+  const limit = pLimit(25);
 
   while (true) {
     const queries: string[] = [Query.limit(batchSize)];
@@ -729,6 +754,16 @@ async function copyAttributeData(
 
     totalCopied += docs.length;
     lastId = docs[docs.length - 1].$id;
+
+    // Appwrite caps result.total at 5000 — adjust progress bar if we exceed it
+    if (progress && totalDocs && totalCopied > totalDocs) {
+      // Estimate remaining: if we haven't hit the last page, assume at least one more batch
+      const estimatedTotal = docs.length < batchSize
+        ? totalCopied
+        : totalCopied + batchSize;
+      progress.setTotal(estimatedTotal);
+      totalDocs = estimatedTotal;
+    }
     progress?.update(totalCopied);
 
     if (docs.length < batchSize) break; // last page
@@ -965,6 +1000,7 @@ function loadOrCreateCheckpoint(
 
   const now = new Date().toISOString();
   return {
+    version: 2,
     planFile,
     startedAt: now,
     lastUpdatedAt: now,
@@ -1026,11 +1062,9 @@ const PHASE_ORDER: CheckpointPhase[] = [
   "backup_created",
   "data_copied_to_backup",
   "data_verified_backup",
-  "original_deleted",
-  "new_attr_created",
-  "data_copied_back",
-  "data_verified_final",
-  "backup_deleted",
+  "original_cleared",
+  "backup_renamed",
+  "indexes_recreated",
   "completed",
 ];
 
@@ -1045,8 +1079,13 @@ function phaseIndex(phase: CheckpointPhase): number {
 
 function printDryRunSummary(plan: MigrationPlan): void {
   console.log("");
-  console.log(chalk.bold("Dry Run — What Would Happen:"));
+  console.log(chalk.bold("Dry Run — What Would Happen (rename-based flow):"));
   console.log(chalk.gray("─".repeat(50)));
+  console.log(
+    chalk.dim(
+      "  Flow: create mig_X (target type) → copy data → delete/archive original → rename mig_X → X"
+    )
+  );
 
   const groups = new Map<string, MigrationPlanEntry[]>();
   for (const entry of plan.entries) {

package/src/migrations/migrateStringsTypes.ts

@@ -62,13 +62,17 @@ export const CheckpointPhase = z.enum([
   "backup_created",
   "data_copied_to_backup",
   "data_verified_backup",
+  "original_cleared",
+  "backup_renamed",
+  "indexes_recreated",
+  "completed",
+  "failed",
+  // Legacy phases (kept for Zod validation of old v1 checkpoints)
   "original_deleted",
   "new_attr_created",
   "data_copied_back",
   "data_verified_final",
   "backup_deleted",
-  "completed",
-  "failed",
 ]);
 export type CheckpointPhase = z.infer<typeof CheckpointPhase>;
 
@@ -96,6 +100,7 @@ export const CheckpointEntrySchema = z.object({
 export type CheckpointEntry = z.infer<typeof CheckpointEntrySchema>;
 
 export const MigrationCheckpointSchema = z.object({
+  version: z.number().default(1),
   planFile: z.string(),
   startedAt: z.string(),
   lastUpdatedAt: z.string(),
@@ -152,6 +157,19 @@ export function generateBackupKey(originalKey: string): string {
   return `${TRUNC_PREFIX}${originalKey.slice(0, maxOrigLen)}_${hash}`;
 }
 
+const ARCHIVE_PREFIX = "og_";
+
+export function generateArchiveKey(originalKey: string): string {
+  const candidate = `${ARCHIVE_PREFIX}${originalKey}`;
+  if (candidate.length <= MAX_KEY_LENGTH) {
+    return candidate;
+  }
+  const hash = simpleHash(originalKey);
+  const TRUNC_PREFIX = "o_";
+  const maxOrigLen = MAX_KEY_LENGTH - TRUNC_PREFIX.length - 1 - 4;
+  return `${TRUNC_PREFIX}${originalKey.slice(0, maxOrigLen)}_${hash}`;
+}
+
 function simpleHash(str: string): string {
   let h = 0;
   for (let i = 0; i < str.length; i++) {