appwrite-utils-cli 1.4.1 → 1.5.0

package/src/collections/methods.ts

@@ -6,10 +6,10 @@ import {
   Query,
   type Models,
 } from "node-appwrite";
-import type { AppwriteConfig, CollectionCreate, Indexes } from "appwrite-utils";
-import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
-import { getAdapterFromConfig } from "../utils/getClientFromConfig.js";
-import { nameToIdMapping, processQueue, queuedOperations } from "../shared/operationQueue.js";
+import type { AppwriteConfig, CollectionCreate, Indexes, Attribute } from "appwrite-utils";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
+import { getAdapterFromConfig } from "../utils/getClientFromConfig.js";
+import { nameToIdMapping, processQueue, queuedOperations } from "../shared/operationQueue.js";
 import { createUpdateCollectionAttributes, createUpdateCollectionAttributesWithStatusCheck } from "./attributes.js";
 import { createOrUpdateIndexes, createOrUpdateIndexesWithStatusCheck } from "./indexes.js";
 import { SchemaGenerator } from "../shared/schemaGenerator.js";
@@ -290,11 +290,11 @@ export const wipeDatabase = async (
   return collectionsDeleted;
 };
 
-export const wipeCollection = async (
-  database: Databases,
-  databaseId: string,
-  collectionId: string
-): Promise<void> => {
+export const wipeCollection = async (
+  database: Databases,
+  databaseId: string,
+  collectionId: string
+): Promise<void> => {
   const collections = await database.listCollections(databaseId, [
     Query.equal("$id", collectionId),
   ]);
@@ -303,8 +303,78 @@ export const wipeCollection = async (
     return;
   }
   const collection = collections.collections[0];
-  await wipeDocumentsFromCollection(database, databaseId, collection.$id);
-};
+  await wipeDocumentsFromCollection(database, databaseId, collection.$id);
+};
+
+// TablesDB helpers for wiping
+export const wipeAllTables = async (
+  adapter: DatabaseAdapter,
+  databaseId: string
+): Promise<{ tableId: string; tableName: string }[]> => {
+  MessageFormatter.info(`Wiping tables in database: ${databaseId}`, { prefix: 'Wipe' });
+  const res = await adapter.listTables({ databaseId, queries: [Query.limit(500)] });
+  const tables: any[] = (res as any).tables || [];
+  const deleted: { tableId: string; tableName: string }[] = [];
+  const progress = ProgressManager.create(`wipe-db-${databaseId}`, tables.length, { title: 'Deleting tables' });
+  let processed = 0;
+  for (const t of tables) {
+    try {
+      await adapter.deleteTable({ databaseId, tableId: t.$id });
+      deleted.push({ tableId: t.$id, tableName: t.name });
+    } catch (e) {
+      MessageFormatter.error(`Failed deleting table ${t.$id}`, e instanceof Error ? e : new Error(String(e)), { prefix: 'Wipe' });
+    }
+    processed++; progress.update(processed);
+    await delay(100);
+  }
+  progress.stop();
+  return deleted;
+};
+
+export const wipeTableRows = async (
+  adapter: DatabaseAdapter,
+  databaseId: string,
+  tableId: string
+): Promise<void> => {
+  try {
+    const initial = await adapter.listRows({ databaseId, tableId, queries: [Query.limit(1000)] });
+    let rows: any[] = (initial as any).rows || [];
+    let total = rows.length;
+    let cursor = rows.length >= 1000 ? rows[rows.length - 1].$id : undefined;
+    while (cursor) {
+      const resp = await adapter.listRows({ databaseId, tableId, queries: [Query.limit(1000), ...(cursor ? [Query.cursorAfter(cursor)] : [])] });
+      const more: any[] = (resp as any).rows || [];
+      rows.push(...more);
+      total = rows.length;
+      cursor = more.length >= 1000 ? more[more.length - 1].$id : undefined;
+      if (total % 10000 === 0) {
+        MessageFormatter.progress(`Found ${total} rows...`, { prefix: 'Wipe' });
+      }
+    }
+    MessageFormatter.info(`Found ${total} rows to delete`, { prefix: 'Wipe' });
+    if (total === 0) return;
+    const progress = ProgressManager.create(`delete-${tableId}`, total, { title: 'Deleting rows' });
+    let processed = 0;
+    const maxStackSize = 50;
+    const batches = chunk(rows, maxStackSize);
+    for (const batch of batches) {
+      await Promise.all(batch.map(async (row: any) => {
+        try {
+          await adapter.deleteRow({ databaseId, tableId, id: row.$id });
+        } catch (e: any) {
+          // ignore missing rows
+        }
+        processed++; progress.update(processed);
+      }));
+      await delay(50);
+    }
+    progress.stop();
+    MessageFormatter.success(`Completed deletion of ${total} rows from table ${tableId}`, { prefix: 'Wipe' });
+  } catch (error) {
+    MessageFormatter.error(`Error wiping rows from table ${tableId}`, error instanceof Error ? error : new Error(String(error)), { prefix: 'Wipe' });
+    throw error;
+  }
+};
 
 export const generateSchemas = async (
   config: AppwriteConfig,
@@ -314,13 +384,23 @@ export const generateSchemas = async (
   schemaGenerator.generateSchemas();
 };
 
-export const createOrUpdateCollections = async (
-  database: Databases,
-  databaseId: string,
-  config: AppwriteConfig,
-  deletedCollections?: { collectionId: string; collectionName: string }[],
-  selectedCollections: Models.Collection[] = []
-): Promise<void> => {
+export const createOrUpdateCollections = async (
+  database: Databases,
+  databaseId: string,
+  config: AppwriteConfig,
+  deletedCollections?: { collectionId: string; collectionName: string }[],
+  selectedCollections: Models.Collection[] = []
+): Promise<void> => {
+  // If API mode is tablesdb, route to adapter-based implementation
+  try {
+    const { adapter, apiMode } = await getAdapterFromConfig(config);
+    if (apiMode === 'tablesdb') {
+      await createOrUpdateCollectionsViaAdapter(adapter, databaseId, config, deletedCollections, selectedCollections);
+      return;
+    }
+  } catch {
+    // Fallback to legacy path below
+  }
   const collectionsToProcess =
     selectedCollections.length > 0 ? selectedCollections : config.collections;
   if (!collectionsToProcess) {
@@ -472,7 +552,200 @@ export const createOrUpdateCollections = async (
   } else {
     MessageFormatter.info("No queued operations to process", { prefix: "Collections" });
   }
-};
+};
+
+// New: Adapter-based implementation for TablesDB
+export const createOrUpdateCollectionsViaAdapter = async (
+  adapter: DatabaseAdapter,
+  databaseId: string,
+  config: AppwriteConfig,
+  deletedCollections?: { collectionId: string; collectionName: string }[],
+  selectedCollections: Models.Collection[] = []
+): Promise<void> => {
+  const collectionsToProcess =
+    selectedCollections.length > 0 ? selectedCollections : (config.collections || []);
+  if (!collectionsToProcess || collectionsToProcess.length === 0) return;
+
+  const usedIds = new Set<string>();
+
+  // Helper: create attributes through adapter
+  const createAttr = async (tableId: string, attr: Attribute) => {
+    const base: any = {
+      databaseId,
+      tableId,
+      key: attr.key,
+      type: (attr as any).type,
+      size: (attr as any).size,
+      required: !!(attr as any).required,
+      default: (attr as any).xdefault,
+      array: !!(attr as any).array,
+      min: (attr as any).min,
+      max: (attr as any).max,
+      elements: (attr as any).elements,
+      encrypt: (attr as any).encrypted,
+      relatedCollection: (attr as any).relatedCollection,
+      relationType: (attr as any).relationType,
+      twoWay: (attr as any).twoWay,
+      twoWayKey: (attr as any).twoWayKey,
+      onDelete: (attr as any).onDelete,
+      side: (attr as any).side,
+    };
+    await adapter.createAttribute(base);
+    await delay(150);
+  };
+
+  // Local queue for unresolved relationships
+  const relQueue: { tableId: string; attr: Attribute }[] = [];
+
+  for (const collection of collectionsToProcess) {
+    const { attributes, indexes, ...collectionData } = collection as any;
+
+    // Prepare permissions as strings (reuse Permission helper)
+    const permissions: string[] = [];
+    if (collection.$permissions && collection.$permissions.length > 0) {
+      for (const p of collection.$permissions as any[]) {
+        if (typeof p === 'string') permissions.push(p);
+        else {
+          switch (p.permission) {
+            case 'read': permissions.push(Permission.read(p.target)); break;
+            case 'create': permissions.push(Permission.create(p.target)); break;
+            case 'update': permissions.push(Permission.update(p.target)); break;
+            case 'delete': permissions.push(Permission.delete(p.target)); break;
+            case 'write': permissions.push(Permission.write(p.target)); break;
+            default: break;
+          }
+        }
+      }
+    }
+
+    // Find existing table by name
+    const list = await adapter.listTables({ databaseId, queries: [Query.equal('name', collectionData.name)] });
+    const items: any[] = (list as any).tables || [];
+    let table = items[0];
+    let tableId: string;
+
+    if (!table) {
+      // Determine ID (prefer provided $id or re-use deleted one)
+      let foundColl = deletedCollections?.find(
+        (coll) => coll.collectionName.toLowerCase().trim().replace(" ", "") === collectionData.name.toLowerCase().trim().replace(" ", "")
+      );
+      if (collectionData.$id) tableId = collectionData.$id;
+      else if (foundColl && !usedIds.has(foundColl.collectionId)) tableId = foundColl.collectionId;
+      else tableId = ID.unique();
+      usedIds.add(tableId);
+
+      const res = await adapter.createTable({
+        databaseId,
+        id: tableId,
+        name: collectionData.name,
+        permissions,
+        documentSecurity: !!collectionData.documentSecurity,
+        enabled: collectionData.enabled !== false
+      });
+      table = (res as any).data || res;
+      nameToIdMapping.set(collectionData.name, tableId);
+    } else {
+      tableId = table.$id;
+      await adapter.updateTable({
+        databaseId,
+        id: tableId,
+        name: collectionData.name,
+        permissions,
+        documentSecurity: !!collectionData.documentSecurity,
+        enabled: collectionData.enabled !== false
+      });
+    }
+
+    // Add small delay after table create/update
+    await delay(250);
+
+    // Create attributes: non-relationship first
+    const nonRel = (attributes || []).filter((a: Attribute) => a.type !== 'relationship');
+    for (const attr of nonRel) {
+      await createAttr(tableId, attr as Attribute);
+    }
+
+    // Relationship attributes — resolve relatedCollection to ID
+    const rels = (attributes || []).filter((a: Attribute) => a.type === 'relationship');
+    for (const attr of rels as any[]) {
+      const relNameOrId = attr.relatedCollection as string | undefined;
+      if (!relNameOrId) continue;
+      let relId = nameToIdMapping.get(relNameOrId) || relNameOrId;
+
+      // If looks like a name (not ULID) and not in cache, try query by name
+      if (!nameToIdMapping.has(relNameOrId)) {
+        try {
+          const relList = await adapter.listTables({ databaseId, queries: [Query.equal('name', relNameOrId)] });
+          const relItems: any[] = (relList as any).tables || [];
+          if (relItems[0]?.$id) {
+            relId = relItems[0].$id;
+            nameToIdMapping.set(relNameOrId, relId);
+          }
+        } catch {}
+      }
+
+      if (relId && typeof relId === 'string') {
+        attr.relatedCollection = relId;
+        await createAttr(tableId, attr as Attribute);
+      } else {
+        // Defer if unresolved
+        relQueue.push({ tableId, attr: attr as Attribute });
+      }
+    }
+
+    // Indexes
+    const idxs = (indexes || []) as any[];
+    for (const idx of idxs) {
+      try {
+        await adapter.createIndex({
+          databaseId,
+          tableId,
+          key: idx.key,
+          type: idx.type,
+          attributes: idx.attributes,
+          orders: idx.orders || []
+        });
+        await delay(150);
+      } catch (e) {
+        MessageFormatter.error(`Failed to create index ${idx.key}`, e instanceof Error ? e : new Error(String(e)), { prefix: 'Indexes' });
+      }
+    }
+  }
+
+  // Process queued relationships once mapping likely populated
+  for (const { tableId, attr } of relQueue) {
+    const relNameOrId = (attr as any).relatedCollection as string | undefined;
+    if (!relNameOrId) continue;
+    const relId = nameToIdMapping.get(relNameOrId) || relNameOrId;
+    if (relId) {
+      (attr as any).relatedCollection = relId;
+      try {
+        await adapter.createAttribute({
+          databaseId,
+          tableId,
+          key: (attr as any).key,
+          type: (attr as any).type,
+          size: (attr as any).size,
+          required: !!(attr as any).required,
+          default: (attr as any).xdefault,
+          array: !!(attr as any).array,
+          min: (attr as any).min,
+          max: (attr as any).max,
+          elements: (attr as any).elements,
+          relatedCollection: relId,
+          relationType: (attr as any).relationType,
+          twoWay: (attr as any).twoWay,
+          twoWayKey: (attr as any).twoWayKey,
+          onDelete: (attr as any).onDelete,
+          side: (attr as any).side
+        });
+        await delay(150);
+      } catch (e) {
+        MessageFormatter.error(`Failed queued relationship ${attr.key}`, e instanceof Error ? e : new Error(String(e)), { prefix: 'Attributes' });
+      }
+    }
+  }
+};
 
 export const generateMockData = async (
   database: Databases,

package/src/migrations/comprehensiveTransfer.ts

@@ -16,7 +16,8 @@ import {
   AppwriteException,
 } from "node-appwrite";
 import { InputFile } from "node-appwrite/file";
-import { MessageFormatter } from "../shared/messageFormatter.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+import { processQueue, queuedOperations } from "../shared/operationQueue.js";
 import { ProgressManager } from "../shared/progressManager.js";
 import { getClient } from "../utils/getClientFromConfig.js";
 import {
@@ -624,13 +625,26 @@ export class ComprehensiveTransfer {
             { prefix: "Transfer" }
           );
         }
-      }
-    } catch (error) {
-      MessageFormatter.error(
-        `Failed to create database structure for ${dbId}`,
-        error instanceof Error ? error : new Error(String(error)),
-        { prefix: "Transfer" }
-      );
+      }
+      // After processing all collections' attributes and indexes, process any queued
+      // relationship attributes so dependencies are resolved within this phase.
+      if (queuedOperations.length > 0) {
+        MessageFormatter.info(
+          `Processing ${queuedOperations.length} queued relationship operations`,
+          { prefix: "Transfer" }
+        );
+        await processQueue(this.targetDatabases, dbId);
+      } else {
+        MessageFormatter.info("No queued relationship operations to process", {
+          prefix: "Transfer",
+        });
+      }
+    } catch (error) {
+      MessageFormatter.error(
+        `Failed to create database structure for ${dbId}`,
+        error instanceof Error ? error : new Error(String(error)),
+        { prefix: "Transfer" }
+      );
       throw error;
     }
   }

package/src/shared/jsonSchemaGenerator.ts

@@ -33,16 +33,23 @@ export interface JsonSchema {
   definitions?: Record<string, JsonSchemaProperty>;
 }
 
-export class JsonSchemaGenerator {
-  private config: AppwriteConfig;
-  private appwriteFolderPath: string;
-  private relationshipMap = new Map<string, any[]>();
-
-  constructor(config: AppwriteConfig, appwriteFolderPath: string) {
-    this.config = config;
-    this.appwriteFolderPath = appwriteFolderPath;
-    this.extractRelationships();
-  }
+export class JsonSchemaGenerator {
+  private config: AppwriteConfig;
+  private appwriteFolderPath: string;
+  private relationshipMap = new Map<string, any[]>();
+
+  constructor(config: AppwriteConfig, appwriteFolderPath: string) {
+    this.config = config;
+    this.appwriteFolderPath = appwriteFolderPath;
+    this.extractRelationships();
+  }
+
+  private resolveCollectionName = (idOrName: string): string => {
+    const col = this.config.collections?.find(
+      (c) => c.$id === (idOrName as any) || c.name === idOrName
+    );
+    return col?.name ?? idOrName;
+  };
 
   private extractRelationships(): void {
     if (!this.config.collections) return;
@@ -51,16 +58,16 @@ export class JsonSchemaGenerator {
       if (!collection.attributes) return;
 
       collection.attributes.forEach((attr) => {
-        if (attr.type === "relationship" && attr.relatedCollection) {
-          const relationships = this.relationshipMap.get(collection.name) || [];
-          relationships.push({
-            attributeKey: attr.key,
-            relatedCollection: attr.relatedCollection,
-            relationType: attr.relationType,
-            isArray: attr.relationType === "oneToMany" || attr.relationType === "manyToMany"
-          });
-          this.relationshipMap.set(collection.name, relationships);
-        }
+        if (attr.type === "relationship" && attr.relatedCollection) {
+          const relationships = this.relationshipMap.get(collection.name) || [];
+          relationships.push({
+            attributeKey: attr.key,
+            relatedCollection: this.resolveCollectionName(attr.relatedCollection),
+            relationType: attr.relationType,
+            isArray: attr.relationType === "oneToMany" || attr.relationType === "manyToMany"
+          });
+          this.relationshipMap.set(collection.name, relationships);
+        }
       });
     });
   }
@@ -153,14 +160,14 @@ export class JsonSchemaGenerator {
         }
         break;
 
-      case "relationship":
-        if (attribute.relatedCollection) {
-          // For relationships, reference the related collection schema
-          schema.$ref = `#/definitions/${toPascalCase(attribute.relatedCollection)}`;
-        } else {
-          schema.type = "string";
-        }
-        break;
+      case "relationship":
+        if (attribute.relatedCollection) {
+          // For relationships, reference the related collection schema
+          schema.$ref = `#/definitions/${toPascalCase(this.resolveCollectionName(attribute.relatedCollection))}`;
+        } else {
+          schema.type = "string";
+        }
+        break;
 
       default:
         schema.type = "string";
@@ -385,4 +392,4 @@ export default jsonSchemas;
 
     return { valid: errors.length === 0, errors };
   }
-}
+}

package/src/shared/operationQueue.ts

@@ -33,39 +33,54 @@ export const processQueue = async (db: Databases, dbId: string) => {
       let collectionFound: Models.Collection | undefined;
 
       // Handle relationship attribute operations
-      if (operation.attribute?.type === "relationship") {
-        // Attempt to resolve the collection directly if collectionId is specified
-        if (operation.collectionId) {
-          console.log(`\tFetching collection by ID: ${operation.collectionId}`);
-          try {
-            collectionFound = await tryAwaitWithRetry(
-              async () => await db.getCollection(dbId, operation.collectionId!)
-            );
-          } catch (e) {
-            console.log(
-              `\tCollection not found by ID: ${operation.collectionId}`
-            );
-          }
-        }
-        // Attempt to resolve related collection if specified and not already found
-        if (!collectionFound && operation.attribute?.relatedCollection) {
-          collectionFound = await fetchAndCacheCollectionByName(
-            db,
-            dbId,
-            operation.attribute.relatedCollection
-          );
-        }
-        // Handle dependencies if collection still not found
-        if (!collectionFound) {
-          for (const dep of operation.dependencies || []) {
-            collectionFound = await fetchAndCacheCollectionByName(
-              db,
-              dbId,
-              dep
-            );
-            if (collectionFound) break; // Break early if collection is found
-          }
-        }
+      if (operation.attribute?.type === "relationship") {
+        // Attempt to resolve the collection directly if collectionId is specified
+        if (operation.collectionId) {
+          console.log(`\tFetching collection by ID: ${operation.collectionId}`);
+          try {
+            collectionFound = await tryAwaitWithRetry(
+              async () => await db.getCollection(dbId, operation.collectionId!)
+            );
+          } catch (e) {
+            console.log(
+              `\tCollection not found by ID: ${operation.collectionId}`
+            );
+          }
+        }
+        // Attempt to resolve related collection if specified and not already found
+        if (!collectionFound && operation.attribute?.relatedCollection) {
+          // First, try treating relatedCollection as an ID
+          try {
+            const relAttr: any = operation.attribute as any;
+            const byId = await tryAwaitWithRetry(
+              async () => await db.getCollection(dbId, relAttr.relatedCollection as string)
+            );
+            // We still need the target collection (operation.collectionId) to create the attribute on,
+            // so only use this branch to warm caches/mappings and continue to dependency checks.
+            // Do not override collectionFound with the related collection.
+          } catch (_) {
+            // Not an ID or not found; fall back to name-based cache
+          }
+
+          // Warm cache by name (used by attribute creation path), but do not use as target collection
+          const relAttr: any = operation.attribute as any;
+          await fetchAndCacheCollectionByName(
+            db,
+            dbId,
+            relAttr.relatedCollection
+          );
+        }
+        // Handle dependencies if collection still not found
+        if (!collectionFound) {
+          for (const dep of operation.dependencies || []) {
+            collectionFound = await fetchAndCacheCollectionByName(
+              db,
+              dbId,
+              dep
+            );
+            if (collectionFound) break; // Break early if collection is found
+          }
+        }
       } else if (operation.collectionId) {
         // Handle non-relationship operations with a specified collectionId
         console.log(`\tFetching collection by ID: ${operation.collectionId}`);