appwrite-utils-cli 1.7.9 → 1.8.2

package/src/migrations/comprehensiveTransfer.ts

@@ -18,8 +18,8 @@ import {
 import { InputFile } from "node-appwrite/file";
 import { MessageFormatter } from "../shared/messageFormatter.js";
 import { processQueue, queuedOperations } from "../shared/operationQueue.js";
-import { ProgressManager } from "../shared/progressManager.js";
-import { getClient } from "../utils/getClientFromConfig.js";
+import { ProgressManager } from "../shared/progressManager.js";
+import { getClient } from "../utils/getClientFromConfig.js";
 import {
   transferDatabaseLocalToLocal,
   transferDatabaseLocalToRemote,
@@ -35,7 +35,10 @@ import {
 import pLimit from "p-limit";
 import chalk from "chalk";
 import { join } from "node:path";
-import fs from "node:fs";
+import fs from "node:fs";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
+import { getAdapter } from "../utils/getClientFromConfig.js";
+import { mapToCreateAttributeParams } from "../shared/attributeMapper.js";
 
 export interface ComprehensiveTransferOptions {
   sourceEndpoint: string;
@@ -78,10 +81,12 @@ export class ComprehensiveTransfer {
   private limit: ReturnType<typeof pLimit>;
   private userLimit: ReturnType<typeof pLimit>;
   private fileLimit: ReturnType<typeof pLimit>;
-  private results: TransferResults;
-  private startTime: number;
-  private tempDir: string;
-  private cachedMaxFileSize?: number; // Cache successful maximumFileSize for subsequent buckets
+  private results: TransferResults;
+  private startTime: number;
+  private tempDir: string;
+  private cachedMaxFileSize?: number; // Cache successful maximumFileSize for subsequent buckets
+  private sourceAdapter?: DatabaseAdapter;
+  private targetAdapter?: DatabaseAdapter;
 
   constructor(private options: ComprehensiveTransferOptions) {
     this.sourceClient = getClient(
@@ -126,11 +131,27 @@ export class ComprehensiveTransfer {
     this.tempDir = join(process.cwd(), ".appwrite-transfer-temp");
   }
 
-  async execute(): Promise<TransferResults> {
-    try {
-      MessageFormatter.info("Starting comprehensive transfer", {
-        prefix: "Transfer",
-      });
+  async execute(): Promise<TransferResults> {
+    try {
+      MessageFormatter.info("Starting comprehensive transfer", {
+        prefix: "Transfer",
+      });
+
+      // Initialize adapters for unified API (TablesDB or legacy via adapter)
+      const source = await getAdapter(
+        this.options.sourceEndpoint,
+        this.options.sourceProject,
+        this.options.sourceKey,
+        'auto'
+      );
+      const target = await getAdapter(
+        this.options.targetEndpoint,
+        this.options.targetProject,
+        this.options.targetKey,
+        'auto'
+      );
+      this.sourceAdapter = source.adapter;
+      this.targetAdapter = target.adapter;
 
       if (this.options.dryRun) {
         MessageFormatter.info("DRY RUN MODE - No actual changes will be made", {
@@ -1405,48 +1426,103 @@ export class ComprehensiveTransfer {
   /**
    * Helper method to create collection attributes with status checking
    */
-  private async createCollectionAttributesWithStatusCheck(
-    databases: Databases,
-    dbId: string,
-    collection: Models.Collection,
-    attributes: any[]
-  ): Promise<boolean> {
-    // Import the enhanced attribute creation function
-    const { createUpdateCollectionAttributesWithStatusCheck } = await import(
-      "../collections/attributes.js"
-    );
-
-    return await createUpdateCollectionAttributesWithStatusCheck(
-      databases,
-      dbId,
-      collection,
-      attributes
-    );
-  }
+  private async createCollectionAttributesWithStatusCheck(
+    databases: Databases,
+    dbId: string,
+    collection: Models.Collection,
+    attributes: any[]
+  ): Promise<boolean> {
+    if (!this.targetAdapter) {
+      throw new Error('Target adapter not initialized');
+    }
+
+    try {
+      // Create non-relationship attributes first
+      const nonRel = (attributes || []).filter((a: any) => a.type !== 'relationship');
+      for (const attr of nonRel) {
+        const params = mapToCreateAttributeParams(attr as any, { databaseId: dbId, tableId: collection.$id });
+        await this.targetAdapter.createAttribute(params);
+        // Small delay between creations
+        await new Promise((r) => setTimeout(r, 150));
+      }
+
+      // Wait for attributes to become available
+      for (const attr of nonRel) {
+        const maxWait = 60000; // 60s
+        const start = Date.now();
+        let lastStatus = '';
+        while (Date.now() - start < maxWait) {
+          try {
+            const tableRes = await this.targetAdapter.getTable({ databaseId: dbId, tableId: collection.$id });
+            const cols = (tableRes as any).attributes || (tableRes as any).columns || [];
+            const col = cols.find((c: any) => c.key === attr.key);
+            if (col) {
+              if (col.status === 'available') break;
+              if (col.status === 'failed' || col.status === 'stuck') {
+                throw new Error(col.error || `Attribute ${attr.key} failed`);
+              }
+              lastStatus = col.status;
+            }
+            await new Promise((r) => setTimeout(r, 2000));
+          } catch {
+            await new Promise((r) => setTimeout(r, 2000));
+          }
+        }
+        if (Date.now() - start >= maxWait) {
+          MessageFormatter.warning(
+            `Attribute ${attr.key} did not become available within 60s (last status: ${lastStatus})`,
+            { prefix: 'Attributes' }
+          );
+        }
+      }
+
+      // Create relationship attributes
+      const rels = (attributes || []).filter((a: any) => a.type === 'relationship');
+      for (const attr of rels) {
+        const params = mapToCreateAttributeParams(attr as any, { databaseId: dbId, tableId: collection.$id });
+        await this.targetAdapter.createAttribute(params);
+        await new Promise((r) => setTimeout(r, 150));
+      }
+
+      return true;
+    } catch (e) {
+      MessageFormatter.error('Failed creating attributes via adapter', e instanceof Error ? e : new Error(String(e)), { prefix: 'Attributes' });
+      return false;
+    }
+  }
 
   /**
    * Helper method to create collection indexes with status checking
    */
-  private async createCollectionIndexesWithStatusCheck(
-    dbId: string,
-    databases: Databases,
-    collectionId: string,
-    collection: Models.Collection,
-    indexes: any[]
-  ): Promise<boolean> {
-    // Import the enhanced index creation function
-    const { createOrUpdateIndexesWithStatusCheck } = await import(
-      "../collections/indexes.js"
-    );
-
-    return await createOrUpdateIndexesWithStatusCheck(
-      dbId,
-      databases,
-      collectionId,
-      collection,
-      indexes
-    );
-  }
+  private async createCollectionIndexesWithStatusCheck(
+    dbId: string,
+    databases: Databases,
+    collectionId: string,
+    collection: Models.Collection,
+    indexes: any[]
+  ): Promise<boolean> {
+    if (!this.targetAdapter) {
+      throw new Error('Target adapter not initialized');
+    }
+
+    try {
+      for (const idx of indexes || []) {
+        await this.targetAdapter.createIndex({
+          databaseId: dbId,
+          tableId: collectionId,
+          key: idx.key,
+          type: idx.type,
+          attributes: idx.attributes,
+          orders: idx.orders || []
+        });
+        await new Promise((r) => setTimeout(r, 150));
+      }
+      return true;
+    } catch (e) {
+      MessageFormatter.error('Failed creating indexes via adapter', e instanceof Error ? e : new Error(String(e)), { prefix: 'Indexes' });
+      return false;
+    }
+  }
 
   /**
    * Helper method to transfer documents between databases using bulk operations with content and permission-based filtering

package/src/migrations/dataLoader.ts

@@ -315,7 +315,7 @@ export class DataLoader {
         collection.$id = collectionExists.$id;
         this.config.collections[index] = collectionConfig;
         // Find or create an import operation for the collection
-        const adapter = new LegacyAdapter(this.database);
+        const adapter = new LegacyAdapter(this.database.client);
         const collectionImportOperation = await findOrCreateOperation(
           adapter,
           dbId,
@@ -957,7 +957,7 @@ export class DataLoader {
       this.oldIdToNewIdPerCollectionMap
         .set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
         .get(this.getCollectionKey(collection.name));
-    const adapter = new LegacyAdapter(this.database);
+    const adapter = new LegacyAdapter(this.database.client);
     if (!operationId) {
       const collectionImportOperation = await findOrCreateOperation(
         adapter,
@@ -1188,7 +1188,7 @@ export class DataLoader {
     let operationId = this.collectionImportOperations.get(
       this.getCollectionKey(collection.name)
     );
-    const adapter = new LegacyAdapter(this.database);
+    const adapter = new LegacyAdapter(this.database.client);
     if (!operationId) {
       const collectionImportOperation = await findOrCreateOperation(
         adapter,

package/src/migrations/importController.ts

@@ -289,7 +289,7 @@ export class ImportController {
           "currentOperations",
           importOperationId
         );
-        const adapter = new LegacyAdapter(this.database);
+        const adapter = new LegacyAdapter(this.database.client);
         await updateOperation(adapter, db.$id, importOperation.$id, {
           status: "in_progress",
         });
@@ -349,7 +349,7 @@ export class ImportController {
           await Promise.all(batchPromises);
           MessageFormatter.success(`Completed batch ${i + 1} of ${dataSplit.length}`, { prefix: "Import" });
           if (importOperation) {
-            const adapter = new LegacyAdapter(this.database);
+            const adapter = new LegacyAdapter(this.database.client);
             await updateOperation(adapter, db.$id, importOperation.$id, {
               progress: processedItems,
             });
@@ -357,7 +357,7 @@ export class ImportController {
         }
         // After all batches are processed, update the operation status to completed
         if (importOperation) {
-          const adapter = new LegacyAdapter(this.database);
+          const adapter = new LegacyAdapter(this.database.client);
           await updateOperation(adapter, db.$id, importOperation.$id, {
             status: "completed",
           });

package/src/migrations/services/ImportOrchestrator.ts

@@ -203,7 +203,7 @@ export class ImportOrchestrator {
         this.config.collections[index] = collectionConfig;
 
         // Find or create an import operation for the collection
-        const adapter = new LegacyAdapter(this.database);
+        const adapter = new LegacyAdapter(this.database.client);
         const collectionImportOperation = await findOrCreateOperation(
           adapter,
           dbId,
@@ -501,7 +501,7 @@ export class ImportOrchestrator {
     logger.info(`Importing collection: ${collection.name} (${collectionData.data.length} items)`);
 
     const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
-    const adapter = new LegacyAdapter(this.database);
+    const adapter = new LegacyAdapter(this.database.client);
     if (operationId) {
       await updateOperation(adapter, db.$id, operationId, { status: "in_progress" });
     }
@@ -638,7 +638,7 @@ export class ImportOrchestrator {
     const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
     if (operationId) {
       const updateData = total ? { status, total } : { status };
-      const adapter = new LegacyAdapter(this.database);
+      const adapter = new LegacyAdapter(this.database.client);
       await updateOperation(adapter, db.$id, operationId, updateData);
     }
   }

package/src/migrations/transfer.ts

@@ -10,22 +10,17 @@ import {
 } from "node-appwrite";
 import { InputFile } from "node-appwrite/file";
 import { getAppwriteClient } from "../utils/helperFunctions.js";
-import {
-  createOrUpdateAttribute,
-  createUpdateCollectionAttributes,
-  createUpdateCollectionAttributesWithStatusCheck,
-} from "../collections/attributes.js";
+// Legacy attribute helpers retained only for local-to-local flows if needed
 import { parseAttribute } from "appwrite-utils";
 import chalk from "chalk";
 import { fetchAllCollections } from "../collections/methods.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+import { LegacyAdapter } from "../adapters/LegacyAdapter.js";
 import { ProgressManager } from "../shared/progressManager.js";
-import {
-  createOrUpdateIndex,
-  createOrUpdateIndexes,
-  createOrUpdateIndexesWithStatusCheck,
-} from "../collections/indexes.js";
-import { getClient } from "../utils/getClientFromConfig.js";
+import { getClient, getAdapter } from "../utils/getClientFromConfig.js";
+import { diffTableColumns } from "../collections/tableOperations.js";
+import { mapToCreateAttributeParams } from "../shared/attributeMapper.js";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
 
 export interface TransferOptions {
   fromDb: Models.Database | undefined;
@@ -331,75 +326,83 @@ export const transferDatabaseLocalToLocal = async (
         );
       }
 
-      // Handle attributes with enhanced status checking
-      MessageFormatter.info(
-        `Creating attributes for collection ${collection.name} with enhanced monitoring...`,
-        { prefix: "Transfer" }
-      );
-
-      const allAttributes = collection.attributes.map((attr) =>
-        parseAttribute(attr as any)
-      );
-      const attributeSuccess =
-        await createUpdateCollectionAttributesWithStatusCheck(
-          localDb,
-          targetDbId,
-          targetCollection,
-          allAttributes
-        );
-
-      if (!attributeSuccess) {
-        MessageFormatter.error(
-          `Failed to create all attributes for collection ${collection.name}, skipping to next collection`,
-          undefined,
-          { prefix: "Transfer" }
-        );
-        continue;
-      }
-
-      MessageFormatter.success(
-        `All attributes created successfully for collection ${collection.name}`,
-        { prefix: "Transfer" }
-      );
-
-      // Handle indexes
-      const existingIndexes = await tryAwaitWithRetry(
-        async () => await localDb.listIndexes(targetDbId, targetCollection.$id)
-      );
-
-      for (const index of collection.indexes) {
-        const existingIndex = existingIndexes.indexes.find(
-          (idx) => idx.key === index.key
-        );
-
-        if (!existingIndex) {
-          await tryAwaitWithRetry(async () =>
-            createOrUpdateIndex(
-              targetDbId,
-              localDb,
-              targetCollection.$id,
-              index as any
-            )
-          );
-          MessageFormatter.success(
-            `Index ${index.key} created`,
-            { prefix: "Transfer" }
-          );
-        } else {
-          MessageFormatter.info(
-            `Index ${index.key} exists, checking for updates...`,
-            { prefix: "Transfer" }
-          );
-          await tryAwaitWithRetry(async () =>
-            createOrUpdateIndex(
-              targetDbId,
-              localDb,
-              targetCollection.$id,
-              index as any
-            )
-          );
-        }
-      }
+      // Create attributes via local adapter (wrap the existing client)
+      const localAdapter: DatabaseAdapter = new LegacyAdapter((localDb as any).client);
+      MessageFormatter.info(`Creating attributes for ${collection.name} via adapter...`, { prefix: 'Transfer' });
+      const uniformAttrs = collection.attributes.map((attr) => parseAttribute(attr as any));
+      const nonRel = uniformAttrs.filter((a: any) => a.type !== 'relationship');
+      for (const attr of nonRel) {
+        const params = mapToCreateAttributeParams(attr as any, { databaseId: targetDbId, tableId: targetCollection.$id });
+        await localAdapter.createAttribute(params);
+        await new Promise((r) => setTimeout(r, 150));
+      }
+
+      // Wait for attributes to become available
+      for (const attr of nonRel) {
+        const maxWait = 60000; const start = Date.now();
+        let lastStatus = '';
+        while (Date.now() - start < maxWait) {
+          try {
+            const tableRes = await localAdapter.getTable({ databaseId: targetDbId, tableId: targetCollection.$id });
+            const attrs = (tableRes as any).attributes || (tableRes as any).columns || [];
+            const found = attrs.find((a: any) => a.key === attr.key);
+            if (found) {
+              if (found.status === 'available') break;
+              if (found.status === 'failed' || found.status === 'stuck') {
+                throw new Error(found.error || `Attribute ${attr.key} failed`);
+              }
+              lastStatus = found.status;
+            }
+            await new Promise((r) => setTimeout(r, 2000));
+          } catch {
+            await new Promise((r) => setTimeout(r, 2000));
+          }
+        }
+        if (Date.now() - start >= maxWait) {
+          MessageFormatter.warning(`Attribute ${attr.key} did not become available within 60s (last: ${lastStatus})`, { prefix: 'Transfer' });
+        }
+      }
+
+      // Relationship attributes
+      const rels = uniformAttrs.filter((a: any) => a.type === 'relationship');
+      for (const attr of rels) {
+        const params = mapToCreateAttributeParams(attr as any, { databaseId: targetDbId, tableId: targetCollection.$id });
+        await localAdapter.createAttribute(params);
+        await new Promise((r) => setTimeout(r, 150));
+      }
+
+      // Handle indexes via adapter (create or update)
+      for (const idx of collection.indexes) {
+        try {
+          await localAdapter.createIndex({
+            databaseId: targetDbId,
+            tableId: targetCollection.$id,
+            key: (idx as any).key,
+            type: (idx as any).type,
+            attributes: (idx as any).attributes,
+            orders: (idx as any).orders || []
+          });
+          await new Promise((r) => setTimeout(r, 150));
+          MessageFormatter.success(`Index ${(idx as any).key} created`, { prefix: 'Transfer' });
+        } catch (e) {
+          // Try update path by deleting and recreating if necessary
+          try {
+            await localAdapter.deleteIndex({ databaseId: targetDbId, tableId: targetCollection.$id, key: (idx as any).key });
+            await localAdapter.createIndex({
+              databaseId: targetDbId,
+              tableId: targetCollection.$id,
+              key: (idx as any).key,
+              type: (idx as any).type,
+              attributes: (idx as any).attributes,
+              orders: (idx as any).orders || []
+            });
+            await new Promise((r) => setTimeout(r, 150));
+            MessageFormatter.info(`Index ${(idx as any).key} recreated`, { prefix: 'Transfer' });
+          } catch (e2) {
+            MessageFormatter.error(`Failed to ensure index ${(idx as any).key}`, e2 instanceof Error ? e2 : new Error(String(e2)), { prefix: 'Transfer' });
+          }
+        }
+      }
 
       // Transfer documents
       const { transferDocumentsBetweenDbsLocalToLocal } = await import(
@@ -500,36 +503,54 @@ export const transferDatabaseLocalToRemote = async (
         );
       }
 
-      // Handle attributes with enhanced status checking
-      MessageFormatter.info(
-        `Creating attributes for collection ${collection.name} with enhanced monitoring...`,
-        { prefix: "Transfer" }
-      );
-
-      const attributesToCreate = collection.attributes.map((attr) =>
-        parseAttribute(attr as any)
-      );
-
-      const attributesSuccess =
-        await createUpdateCollectionAttributesWithStatusCheck(
-          remoteDb,
-          toDbId,
-          targetCollection,
-          attributesToCreate
-        );
-
-      if (!attributesSuccess) {
-        MessageFormatter.warning(
-          `Failed to create some attributes for collection ${collection.name}`,
-          { prefix: "Transfer" }
-        );
-        // Continue with the transfer even if some attributes failed
-      } else {
-        MessageFormatter.success(
-          `All attributes created successfully for collection ${collection.name}`,
-          { prefix: "Transfer" }
-        );
-      }
+      // Create/Update attributes via adapter (prefer adapter for remote)
+      const { adapter: remoteAdapter } = await getAdapter(endpoint, projectId, apiKey, 'auto');
+      MessageFormatter.info(`Creating attributes for ${collection.name} via adapter...`, { prefix: 'Transfer' });
+      const uniformAttrs = collection.attributes.map((attr) => parseAttribute(attr as any));
+      const nonRel = uniformAttrs.filter((a: any) => a.type !== 'relationship');
+      if (nonRel.length > 0) {
+        const tableInfo = await (remoteAdapter as DatabaseAdapter).getTable({ databaseId: toDbId, tableId: collection.$id });
+        const existingCols: any[] = (tableInfo as any).columns || (tableInfo as any).attributes || [];
+        const { toCreate, toUpdate } = diffTableColumns(existingCols, nonRel as any);
+        for (const a of toUpdate) { const p = mapToCreateAttributeParams(a as any, { databaseId: toDbId, tableId: collection.$id }); await (remoteAdapter as DatabaseAdapter).updateAttribute(p as any); await new Promise((r)=>setTimeout(r,150)); }
+        for (const a of toCreate) { const p = mapToCreateAttributeParams(a as any, { databaseId: toDbId, tableId: collection.$id }); await (remoteAdapter as DatabaseAdapter).createAttribute(p); await new Promise((r)=>setTimeout(r,150)); }
+      }
+
+      // Wait for non-relationship attributes to become available
+      for (const attr of nonRel) {
+        const maxWait = 60000; const start = Date.now();
+        let lastStatus = '';
+        while (Date.now() - start < maxWait) {
+          try {
+            const tableRes = await (remoteAdapter as DatabaseAdapter).getTable({ databaseId: toDbId, tableId: collection.$id });
+            const attrs = (tableRes as any).attributes || (tableRes as any).columns || [];
+            const found = attrs.find((a: any) => a.key === attr.key);
+            if (found) {
+              if (found.status === 'available') break;
+              if (found.status === 'failed' || found.status === 'stuck') {
+                throw new Error(found.error || `Attribute ${attr.key} failed`);
+              }
+              lastStatus = found.status;
+            }
+            await new Promise((r) => setTimeout(r, 2000));
+          } catch {
+            await new Promise((r) => setTimeout(r, 2000));
+          }
+        }
+        if (Date.now() - start >= maxWait) {
+          MessageFormatter.warning(`Attribute ${attr.key} did not become available within 60s (last: ${lastStatus})`, { prefix: 'Transfer' });
+        }
+      }
+
+      // Relationship attributes
+      const rels = uniformAttrs.filter((a: any) => a.type === 'relationship');
+      if (rels.length > 0) {
+        const tableInfo2 = await (remoteAdapter as DatabaseAdapter).getTable({ databaseId: toDbId, tableId: collection.$id });
+        const existingCols2: any[] = (tableInfo2 as any).columns || (tableInfo2 as any).attributes || [];
+        const { toCreate: rCreate, toUpdate: rUpdate } = diffTableColumns(existingCols2, rels as any);
+        for (const a of rUpdate) { const p = mapToCreateAttributeParams(a as any, { databaseId: toDbId, tableId: collection.$id }); await (remoteAdapter as DatabaseAdapter).updateAttribute(p as any); await new Promise((r)=>setTimeout(r,150)); }
+        for (const a of rCreate) { const p = mapToCreateAttributeParams(a as any, { databaseId: toDbId, tableId: collection.$id }); await (remoteAdapter as DatabaseAdapter).createAttribute(p); await new Promise((r)=>setTimeout(r,150)); }
+      }
 
       // Handle indexes with enhanced status checking
       MessageFormatter.info(
@@ -537,26 +558,22 @@ export const transferDatabaseLocalToRemote = async (
         { prefix: "Transfer" }
       );
 
-      const indexesSuccess = await createOrUpdateIndexesWithStatusCheck(
-        toDbId,
-        remoteDb,
-        targetCollection.$id,
-        targetCollection,
-        collection.indexes as any
-      );
-
-      if (!indexesSuccess) {
-        MessageFormatter.warning(
-          `Failed to create some indexes for collection ${collection.name}`,
-          { prefix: "Transfer" }
-        );
-        // Continue with the transfer even if some indexes failed
-      } else {
-        MessageFormatter.success(
-          `All indexes created successfully for collection ${collection.name}`,
-          { prefix: "Transfer" }
-        );
-      }
+      // Create indexes via adapter
+      for (const idx of (collection.indexes as any[]) || []) {
+        try {
+          await (remoteAdapter as DatabaseAdapter).createIndex({
+            databaseId: toDbId,
+            tableId: collection.$id,
+            key: idx.key,
+            type: idx.type,
+            attributes: idx.attributes,
+            orders: idx.orders || []
+          });
+          await new Promise((r) => setTimeout(r, 150));
+        } catch (e) {
+          MessageFormatter.error(`Failed to create index ${idx.key}`, e instanceof Error ? e : new Error(String(e)), { prefix: 'Transfer' });
+        }
+      }
 
       // Transfer documents
       const { transferDocumentsBetweenDbsLocalToRemote } = await import(