appwrite-utils-cli 1.4.1 → 1.5.1

package/src/collections/indexes.ts

@@ -1,5 +1,6 @@
-import { indexSchema, type Index } from "appwrite-utils";
-import { Databases, IndexType, Query, type Models } from "node-appwrite";
+import { indexSchema, type Index } from "appwrite-utils";
+import { Databases, IndexType, Query, type Models } from "node-appwrite";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
 import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 import chalk from "chalk";
 
@@ -18,15 +19,15 @@ interface IndexWithStatus {
 /**
  * Wait for index to become available, with retry logic for stuck indexes and exponential backoff
  */
-const waitForIndexAvailable = async (
-  db: Databases,
-  dbId: string,
-  collectionId: string,
-  indexKey: string,
-  maxWaitTime: number = 60000, // 1 minute
-  retryCount: number = 0,
-  maxRetries: number = 5
-): Promise<boolean> => {
+const waitForIndexAvailable = async (
+  db: Databases | DatabaseAdapter,
+  dbId: string,
+  collectionId: string,
+  indexKey: string,
+  maxWaitTime: number = 60000, // 1 minute
+  retryCount: number = 0,
+  maxRetries: number = 5
+): Promise<boolean> => {
   const startTime = Date.now();
   let checkInterval = 2000; // Start with 2 seconds
   
@@ -41,22 +42,29 @@ const waitForIndexAvailable = async (
   
   while (Date.now() - startTime < maxWaitTime) {
     try {
-      const indexList = await db.listIndexes(dbId, collectionId);
-      const index = indexList.indexes.find(
-        (idx: any) => idx.key === indexKey
-      ) as IndexWithStatus | undefined;
+      const indexList = await (db instanceof Databases
+        ? db.listIndexes(dbId, collectionId)
+        : (db as DatabaseAdapter).listIndexes({ databaseId: dbId, tableId: collectionId }));
+      const indexes: any[] = (db instanceof Databases)
+        ? (indexList as any).indexes
+        : ((indexList as any).data || (indexList as any).indexes || []);
+      const index = indexes.find((idx: any) => idx.key === indexKey) as IndexWithStatus | undefined;
       
       if (!index) {
         console.log(chalk.red(`Index '${indexKey}' not found`));
         return false;
       }
       
-      console.log(chalk.gray(`Index '${indexKey}' status: ${index.status}`));
+      if (db instanceof Databases) {
+        console.log(chalk.gray(`Index '${indexKey}' status: ${(index as any).status}`));
+      } else {
+        console.log(chalk.gray(`Index '${indexKey}' detected (TablesDB)`));
+      }
       
       switch (index.status) {
-        case 'available':
-          console.log(chalk.green(`✅ Index '${indexKey}' is now available`));
-          return true;
+        case 'available':
+          console.log(chalk.green(`✅ Index '${indexKey}' is now available`));
+          return true;
           
         case 'failed':
           console.log(chalk.red(`❌ Index '${indexKey}' failed: ${index.error}`));
@@ -66,9 +74,9 @@ const waitForIndexAvailable = async (
           console.log(chalk.yellow(`⚠️ Index '${indexKey}' is stuck, will retry...`));
           return false;
           
-        case 'processing':
-          // Continue waiting
-          break;
+        case 'processing':
+          // Continue waiting
+          break;
           
         case 'deleting':
           console.log(chalk.yellow(`Index '${indexKey}' is being deleted`));
@@ -79,12 +87,12 @@ const waitForIndexAvailable = async (
           break;
       }
       
-      await delay(checkInterval);
-    } catch (error) {
-      console.log(chalk.red(`Error checking index status: ${error}`));
-      return false;
-    }
-  }
+      await delay(checkInterval);
+    } catch (error) {
+      console.log(chalk.red(`Error checking index status: ${error}`));
+      return false;
+    }
+  }
   
   // Timeout reached
   console.log(chalk.yellow(`⏰ Timeout waiting for index '${indexKey}' (${maxWaitTime}ms)`));

package/src/collections/methods.ts

@@ -6,10 +6,10 @@ import {
   Query,
   type Models,
 } from "node-appwrite";
-import type { AppwriteConfig, CollectionCreate, Indexes } from "appwrite-utils";
-import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
-import { getAdapterFromConfig } from "../utils/getClientFromConfig.js";
-import { nameToIdMapping, processQueue, queuedOperations } from "../shared/operationQueue.js";
+import type { AppwriteConfig, CollectionCreate, Indexes, Attribute } from "appwrite-utils";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
+import { getAdapterFromConfig } from "../utils/getClientFromConfig.js";
+import { nameToIdMapping, processQueue, queuedOperations } from "../shared/operationQueue.js";
 import { createUpdateCollectionAttributes, createUpdateCollectionAttributesWithStatusCheck } from "./attributes.js";
 import { createOrUpdateIndexes, createOrUpdateIndexesWithStatusCheck } from "./indexes.js";
 import { SchemaGenerator } from "../shared/schemaGenerator.js";
@@ -290,11 +290,11 @@ export const wipeDatabase = async (
   return collectionsDeleted;
 };
 
-export const wipeCollection = async (
-  database: Databases,
-  databaseId: string,
-  collectionId: string
-): Promise<void> => {
+export const wipeCollection = async (
+  database: Databases,
+  databaseId: string,
+  collectionId: string
+): Promise<void> => {
   const collections = await database.listCollections(databaseId, [
     Query.equal("$id", collectionId),
   ]);
@@ -303,8 +303,78 @@ export const wipeCollection = async (
     return;
   }
   const collection = collections.collections[0];
-  await wipeDocumentsFromCollection(database, databaseId, collection.$id);
-};
+  await wipeDocumentsFromCollection(database, databaseId, collection.$id);
+};
+
+// TablesDB helpers for wiping
+export const wipeAllTables = async (
+  adapter: DatabaseAdapter,
+  databaseId: string
+): Promise<{ tableId: string; tableName: string }[]> => {
+  MessageFormatter.info(`Wiping tables in database: ${databaseId}`, { prefix: 'Wipe' });
+  const res = await adapter.listTables({ databaseId, queries: [Query.limit(500)] });
+  const tables: any[] = (res as any).tables || [];
+  const deleted: { tableId: string; tableName: string }[] = [];
+  const progress = ProgressManager.create(`wipe-db-${databaseId}`, tables.length, { title: 'Deleting tables' });
+  let processed = 0;
+  for (const t of tables) {
+    try {
+      await adapter.deleteTable({ databaseId, tableId: t.$id });
+      deleted.push({ tableId: t.$id, tableName: t.name });
+    } catch (e) {
+      MessageFormatter.error(`Failed deleting table ${t.$id}`, e instanceof Error ? e : new Error(String(e)), { prefix: 'Wipe' });
+    }
+    processed++; progress.update(processed);
+    await delay(100);
+  }
+  progress.stop();
+  return deleted;
+};
+
+export const wipeTableRows = async (
+  adapter: DatabaseAdapter,
+  databaseId: string,
+  tableId: string
+): Promise<void> => {
+  try {
+    const initial = await adapter.listRows({ databaseId, tableId, queries: [Query.limit(1000)] });
+    let rows: any[] = (initial as any).rows || [];
+    let total = rows.length;
+    let cursor = rows.length >= 1000 ? rows[rows.length - 1].$id : undefined;
+    while (cursor) {
+      const resp = await adapter.listRows({ databaseId, tableId, queries: [Query.limit(1000), ...(cursor ? [Query.cursorAfter(cursor)] : [])] });
+      const more: any[] = (resp as any).rows || [];
+      rows.push(...more);
+      total = rows.length;
+      cursor = more.length >= 1000 ? more[more.length - 1].$id : undefined;
+      if (total % 10000 === 0) {
+        MessageFormatter.progress(`Found ${total} rows...`, { prefix: 'Wipe' });
+      }
+    }
+    MessageFormatter.info(`Found ${total} rows to delete`, { prefix: 'Wipe' });
+    if (total === 0) return;
+    const progress = ProgressManager.create(`delete-${tableId}`, total, { title: 'Deleting rows' });
+    let processed = 0;
+    const maxStackSize = 50;
+    const batches = chunk(rows, maxStackSize);
+    for (const batch of batches) {
+      await Promise.all(batch.map(async (row: any) => {
+        try {
+          await adapter.deleteRow({ databaseId, tableId, id: row.$id });
+        } catch (e: any) {
+          // ignore missing rows
+        }
+        processed++; progress.update(processed);
+      }));
+      await delay(50);
+    }
+    progress.stop();
+    MessageFormatter.success(`Completed deletion of ${total} rows from table ${tableId}`, { prefix: 'Wipe' });
+  } catch (error) {
+    MessageFormatter.error(`Error wiping rows from table ${tableId}`, error instanceof Error ? error : new Error(String(error)), { prefix: 'Wipe' });
+    throw error;
+  }
+};
 
 export const generateSchemas = async (
   config: AppwriteConfig,
@@ -314,13 +384,23 @@ export const generateSchemas = async (
   schemaGenerator.generateSchemas();
 };
 
-export const createOrUpdateCollections = async (
-  database: Databases,
-  databaseId: string,
-  config: AppwriteConfig,
-  deletedCollections?: { collectionId: string; collectionName: string }[],
-  selectedCollections: Models.Collection[] = []
-): Promise<void> => {
+export const createOrUpdateCollections = async (
+  database: Databases,
+  databaseId: string,
+  config: AppwriteConfig,
+  deletedCollections?: { collectionId: string; collectionName: string }[],
+  selectedCollections: Models.Collection[] = []
+): Promise<void> => {
+  // If API mode is tablesdb, route to adapter-based implementation
+  try {
+    const { adapter, apiMode } = await getAdapterFromConfig(config);
+    if (apiMode === 'tablesdb') {
+      await createOrUpdateCollectionsViaAdapter(adapter, databaseId, config, deletedCollections, selectedCollections);
+      return;
+    }
+  } catch {
+    // Fallback to legacy path below
+  }
   const collectionsToProcess =
     selectedCollections.length > 0 ? selectedCollections : config.collections;
   if (!collectionsToProcess) {
@@ -472,7 +552,200 @@ export const createOrUpdateCollections = async (
   } else {
     MessageFormatter.info("No queued operations to process", { prefix: "Collections" });
   }
-};
+};
+
+// New: Adapter-based implementation for TablesDB
+export const createOrUpdateCollectionsViaAdapter = async (
+  adapter: DatabaseAdapter,
+  databaseId: string,
+  config: AppwriteConfig,
+  deletedCollections?: { collectionId: string; collectionName: string }[],
+  selectedCollections: Models.Collection[] = []
+): Promise<void> => {
+  const collectionsToProcess =
+    selectedCollections.length > 0 ? selectedCollections : (config.collections || []);
+  if (!collectionsToProcess || collectionsToProcess.length === 0) return;
+
+  const usedIds = new Set<string>();
+
+  // Helper: create attributes through adapter
+  const createAttr = async (tableId: string, attr: Attribute) => {
+    const base: any = {
+      databaseId,
+      tableId,
+      key: attr.key,
+      type: (attr as any).type,
+      size: (attr as any).size,
+      required: !!(attr as any).required,
+      default: (attr as any).xdefault,
+      array: !!(attr as any).array,
+      min: (attr as any).min,
+      max: (attr as any).max,
+      elements: (attr as any).elements,
+      encrypt: (attr as any).encrypted,
+      relatedCollection: (attr as any).relatedCollection,
+      relationType: (attr as any).relationType,
+      twoWay: (attr as any).twoWay,
+      twoWayKey: (attr as any).twoWayKey,
+      onDelete: (attr as any).onDelete,
+      side: (attr as any).side,
+    };
+    await adapter.createAttribute(base);
+    await delay(150);
+  };
+
+  // Local queue for unresolved relationships
+  const relQueue: { tableId: string; attr: Attribute }[] = [];
+
+  for (const collection of collectionsToProcess) {
+    const { attributes, indexes, ...collectionData } = collection as any;
+
+    // Prepare permissions as strings (reuse Permission helper)
+    const permissions: string[] = [];
+    if (collection.$permissions && collection.$permissions.length > 0) {
+      for (const p of collection.$permissions as any[]) {
+        if (typeof p === 'string') permissions.push(p);
+        else {
+          switch (p.permission) {
+            case 'read': permissions.push(Permission.read(p.target)); break;
+            case 'create': permissions.push(Permission.create(p.target)); break;
+            case 'update': permissions.push(Permission.update(p.target)); break;
+            case 'delete': permissions.push(Permission.delete(p.target)); break;
+            case 'write': permissions.push(Permission.write(p.target)); break;
+            default: break;
+          }
+        }
+      }
+    }
+
+    // Find existing table by name
+    const list = await adapter.listTables({ databaseId, queries: [Query.equal('name', collectionData.name)] });
+    const items: any[] = (list as any).tables || [];
+    let table = items[0];
+    let tableId: string;
+
+    if (!table) {
+      // Determine ID (prefer provided $id or re-use deleted one)
+      let foundColl = deletedCollections?.find(
+        (coll) => coll.collectionName.toLowerCase().trim().replace(" ", "") === collectionData.name.toLowerCase().trim().replace(" ", "")
+      );
+      if (collectionData.$id) tableId = collectionData.$id;
+      else if (foundColl && !usedIds.has(foundColl.collectionId)) tableId = foundColl.collectionId;
+      else tableId = ID.unique();
+      usedIds.add(tableId);
+
+      const res = await adapter.createTable({
+        databaseId,
+        id: tableId,
+        name: collectionData.name,
+        permissions,
+        documentSecurity: !!collectionData.documentSecurity,
+        enabled: collectionData.enabled !== false
+      });
+      table = (res as any).data || res;
+      nameToIdMapping.set(collectionData.name, tableId);
+    } else {
+      tableId = table.$id;
+      await adapter.updateTable({
+        databaseId,
+        id: tableId,
+        name: collectionData.name,
+        permissions,
+        documentSecurity: !!collectionData.documentSecurity,
+        enabled: collectionData.enabled !== false
+      });
+    }
+
+    // Add small delay after table create/update
+    await delay(250);
+
+    // Create attributes: non-relationship first
+    const nonRel = (attributes || []).filter((a: Attribute) => a.type !== 'relationship');
+    for (const attr of nonRel) {
+      await createAttr(tableId, attr as Attribute);
+    }
+
+    // Relationship attributes — resolve relatedCollection to ID
+    const rels = (attributes || []).filter((a: Attribute) => a.type === 'relationship');
+    for (const attr of rels as any[]) {
+      const relNameOrId = attr.relatedCollection as string | undefined;
+      if (!relNameOrId) continue;
+      let relId = nameToIdMapping.get(relNameOrId) || relNameOrId;
+
+      // If looks like a name (not ULID) and not in cache, try query by name
+      if (!nameToIdMapping.has(relNameOrId)) {
+        try {
+          const relList = await adapter.listTables({ databaseId, queries: [Query.equal('name', relNameOrId)] });
+          const relItems: any[] = (relList as any).tables || [];
+          if (relItems[0]?.$id) {
+            relId = relItems[0].$id;
+            nameToIdMapping.set(relNameOrId, relId);
+          }
+        } catch {}
+      }
+
+      if (relId && typeof relId === 'string') {
+        attr.relatedCollection = relId;
+        await createAttr(tableId, attr as Attribute);
+      } else {
+        // Defer if unresolved
+        relQueue.push({ tableId, attr: attr as Attribute });
+      }
+    }
+
+    // Indexes
+    const idxs = (indexes || []) as any[];
+    for (const idx of idxs) {
+      try {
+        await adapter.createIndex({
+          databaseId,
+          tableId,
+          key: idx.key,
+          type: idx.type,
+          attributes: idx.attributes,
+          orders: idx.orders || []
+        });
+        await delay(150);
+      } catch (e) {
+        MessageFormatter.error(`Failed to create index ${idx.key}`, e instanceof Error ? e : new Error(String(e)), { prefix: 'Indexes' });
+      }
+    }
+  }
+
+  // Process queued relationships once mapping likely populated
+  for (const { tableId, attr } of relQueue) {
+    const relNameOrId = (attr as any).relatedCollection as string | undefined;
+    if (!relNameOrId) continue;
+    const relId = nameToIdMapping.get(relNameOrId) || relNameOrId;
+    if (relId) {
+      (attr as any).relatedCollection = relId;
+      try {
+        await adapter.createAttribute({
+          databaseId,
+          tableId,
+          key: (attr as any).key,
+          type: (attr as any).type,
+          size: (attr as any).size,
+          required: !!(attr as any).required,
+          default: (attr as any).xdefault,
+          array: !!(attr as any).array,
+          min: (attr as any).min,
+          max: (attr as any).max,
+          elements: (attr as any).elements,
+          relatedCollection: relId,
+          relationType: (attr as any).relationType,
+          twoWay: (attr as any).twoWay,
+          twoWayKey: (attr as any).twoWayKey,
+          onDelete: (attr as any).onDelete,
+          side: (attr as any).side
+        });
+        await delay(150);
+      } catch (e) {
+        MessageFormatter.error(`Failed queued relationship ${attr.key}`, e instanceof Error ? e : new Error(String(e)), { prefix: 'Attributes' });
+      }
+    }
+  }
+};
 
 export const generateMockData = async (
   database: Databases,

package/src/databases/setup.ts

@@ -111,12 +111,13 @@ export const setupMigrationDatabase = async (config: AppwriteConfig) => {
   console.log("---------------------------------");
 };
 
-export const ensureDatabasesExist = async (config: AppwriteConfig, databasesToEnsure?: Models.Database[]) => {
+export const ensureDatabasesExist = async (config: AppwriteConfig, databasesToEnsure?: Models.Database[]) => {
   if (!config.appwriteClient) {
     throw new Error("Appwrite client is not initialized in the config");
   }
-  const database = new Databases(config.appwriteClient);
-  const databasesToCreate = databasesToEnsure || config.databases || [];
+  const database = new Databases(config.appwriteClient);
+  // Work on a shallow copy so we don't mutate caller-provided arrays
+  const databasesToCreate = [...(databasesToEnsure || config.databases || [])];
 
   if (!databasesToCreate.length) {
     console.log("No databases to create");
@@ -130,10 +131,13 @@ export const ensureDatabasesExist = async (config: AppwriteConfig, databasesToEn
   const migrationsDatabase = existingDatabases.databases.find(
     (d) => d.name.toLowerCase().trim().replace(" ", "") === "migrations"
   );
-  if (config.useMigrations && existingDatabases.databases.length !== 0 && migrationsDatabase) {
-    console.log("Creating all databases including migrations");
-    databasesToCreate.push(migrationsDatabase);
-  }
+  if (config.useMigrations && existingDatabases.databases.length !== 0 && migrationsDatabase) {
+    console.log("Creating all databases including migrations");
+    // Ensure migrations exists, but do not mutate the caller's array
+    if (!databasesToCreate.some((d) => d.$id === migrationsDatabase.$id)) {
+      databasesToCreate.push(migrationsDatabase);
+    }
+  }
 
   for (const db of databasesToCreate) {
     if (!existingDatabases.databases.some((d) => d.name === db.name)) {

package/src/interactiveCLI.ts

@@ -1159,13 +1159,14 @@ export class InteractiveCLI {
       return;
     }
 
-    const collections = await this.selectCollections(
-      databases[0],
-      this.controller!.database!,
-      chalk.blue("Select local collections to push:"),
-      true,
-      true // prefer local
-    );
+    const collections = await this.selectCollections(
+      databases[0],
+      this.controller!.database!,
+      chalk.blue("Select local collections to push:"),
+      true,
+      true, // prefer local
+      true  // filter by selected database
+    );
 
     const { syncFunctions } = await inquirer.prompt([
       {

package/src/migrations/comprehensiveTransfer.ts

@@ -16,7 +16,8 @@ import {
   AppwriteException,
 } from "node-appwrite";
 import { InputFile } from "node-appwrite/file";
-import { MessageFormatter } from "../shared/messageFormatter.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+import { processQueue, queuedOperations } from "../shared/operationQueue.js";
 import { ProgressManager } from "../shared/progressManager.js";
 import { getClient } from "../utils/getClientFromConfig.js";
 import {
@@ -624,13 +625,26 @@ export class ComprehensiveTransfer {
             { prefix: "Transfer" }
           );
         }
-      }
-    } catch (error) {
-      MessageFormatter.error(
-        `Failed to create database structure for ${dbId}`,
-        error instanceof Error ? error : new Error(String(error)),
-        { prefix: "Transfer" }
-      );
+      }
+      // After processing all collections' attributes and indexes, process any queued
+      // relationship attributes so dependencies are resolved within this phase.
+      if (queuedOperations.length > 0) {
+        MessageFormatter.info(
+          `Processing ${queuedOperations.length} queued relationship operations`,
+          { prefix: "Transfer" }
+        );
+        await processQueue(this.targetDatabases, dbId);
+      } else {
+        MessageFormatter.info("No queued relationship operations to process", {
+          prefix: "Transfer",
+        });
+      }
+    } catch (error) {
+      MessageFormatter.error(
+        `Failed to create database structure for ${dbId}`,
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Transfer" }
+      );
       throw error;
     }
   }

package/src/shared/jsonSchemaGenerator.ts

@@ -33,16 +33,23 @@ export interface JsonSchema {
   definitions?: Record<string, JsonSchemaProperty>;
 }
 
-export class JsonSchemaGenerator {
-  private config: AppwriteConfig;
-  private appwriteFolderPath: string;
-  private relationshipMap = new Map<string, any[]>();
-
-  constructor(config: AppwriteConfig, appwriteFolderPath: string) {
-    this.config = config;
-    this.appwriteFolderPath = appwriteFolderPath;
-    this.extractRelationships();
-  }
+export class JsonSchemaGenerator {
+  private config: AppwriteConfig;
+  private appwriteFolderPath: string;
+  private relationshipMap = new Map<string, any[]>();
+
+  constructor(config: AppwriteConfig, appwriteFolderPath: string) {
+    this.config = config;
+    this.appwriteFolderPath = appwriteFolderPath;
+    this.extractRelationships();
+  }
+
+  private resolveCollectionName = (idOrName: string): string => {
+    const col = this.config.collections?.find(
+      (c) => c.$id === (idOrName as any) || c.name === idOrName
+    );
+    return col?.name ?? idOrName;
+  };
 
   private extractRelationships(): void {
     if (!this.config.collections) return;
@@ -51,16 +58,16 @@ export class JsonSchemaGenerator {
       if (!collection.attributes) return;
 
       collection.attributes.forEach((attr) => {
-        if (attr.type === "relationship" && attr.relatedCollection) {
-          const relationships = this.relationshipMap.get(collection.name) || [];
-          relationships.push({
-            attributeKey: attr.key,
-            relatedCollection: attr.relatedCollection,
-            relationType: attr.relationType,
-            isArray: attr.relationType === "oneToMany" || attr.relationType === "manyToMany"
-          });
-          this.relationshipMap.set(collection.name, relationships);
-        }
+        if (attr.type === "relationship" && attr.relatedCollection) {
+          const relationships = this.relationshipMap.get(collection.name) || [];
+          relationships.push({
+            attributeKey: attr.key,
+            relatedCollection: this.resolveCollectionName(attr.relatedCollection),
+            relationType: attr.relationType,
+            isArray: attr.relationType === "oneToMany" || attr.relationType === "manyToMany"
+          });
+          this.relationshipMap.set(collection.name, relationships);
+        }
       });
     });
   }
@@ -153,14 +160,14 @@ export class JsonSchemaGenerator {
         }
         break;
 
-      case "relationship":
-        if (attribute.relatedCollection) {
-          // For relationships, reference the related collection schema
-          schema.$ref = `#/definitions/${toPascalCase(attribute.relatedCollection)}`;
-        } else {
-          schema.type = "string";
-        }
-        break;
+      case "relationship":
+        if (attribute.relatedCollection) {
+          // For relationships, reference the related collection schema
+          schema.$ref = `#/definitions/${toPascalCase(this.resolveCollectionName(attribute.relatedCollection))}`;
+        } else {
+          schema.type = "string";
+        }
+        break;
 
       default:
         schema.type = "string";
@@ -385,4 +392,4 @@ export default jsonSchemas;
 
     return { valid: errors.length === 0, errors };
   }
-}
+}