appwrite-utils-cli 1.7.8 → 1.8.1

package/src/collections/transferOperations.ts

@@ -1,120 +1,177 @@
-import {
-  Client,
-  Databases,
-  ID,
-  Query,
-} from "node-appwrite";
-import { tryAwaitWithRetry, delay, calculateExponentialBackoff } from "../utils/helperFunctions.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
-import { chunk } from "es-toolkit";
+import {
+  Client,
+  Databases,
+  ID,
+  Query,
+} from "node-appwrite";
+import { tryAwaitWithRetry, delay, calculateExponentialBackoff } from "../utils/helperFunctions.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+import { chunk } from "es-toolkit";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
+import { isLegacyDatabases } from "../utils/typeGuards.js";
+import { getAdapter } from "../utils/getClientFromConfig.js";
 
 /**
  * Transfers all documents from one collection to another in a different database
  * within the same Appwrite Project
  */
-export const transferDocumentsBetweenDbsLocalToLocal = async (
-  db: Databases,
-  fromDbId: string,
-  toDbId: string,
-  fromCollId: string,
-  toCollId: string
-) => {
-  let fromCollDocs = await tryAwaitWithRetry(async () =>
-    db.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
-  );
-  let totalDocumentsTransferred = 0;
-
-  if (fromCollDocs.documents.length === 0) {
-    MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
-    return;
-  } else if (fromCollDocs.documents.length < 50) {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
-      const toCreateObject: any = {
-        ...doc,
-      };
-      delete toCreateObject.$databaseId;
-      delete toCreateObject.$collectionId;
-      delete toCreateObject.$createdAt;
-      delete toCreateObject.$updatedAt;
-      delete toCreateObject.$id;
-      delete toCreateObject.$permissions;
-      return tryAwaitWithRetry(
-        async () =>
-          await db.createDocument(
-            toDbId,
-            toCollId,
-            doc.$id,
-            toCreateObject,
-            doc.$permissions
-          )
-      );
-    });
-    await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
-  } else {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
-      const toCreateObject: any = {
-        ...doc,
-      };
-      delete toCreateObject.$databaseId;
-      delete toCreateObject.$collectionId;
-      delete toCreateObject.$createdAt;
-      delete toCreateObject.$updatedAt;
-      delete toCreateObject.$id;
-      delete toCreateObject.$permissions;
-      return tryAwaitWithRetry(async () =>
-        db.createDocument(
-          toDbId,
-          toCollId,
-          doc.$id,
-          toCreateObject,
-          doc.$permissions
-        )
-      );
-    });
-    await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
-    while (fromCollDocs.documents.length === 50) {
-      fromCollDocs = await tryAwaitWithRetry(
-        async () =>
-          await db.listDocuments(fromDbId, fromCollId, [
-            Query.limit(50),
-            Query.cursorAfter(
-              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
-            ),
-          ])
-      );
-      const batchedPromises = fromCollDocs.documents.map((doc) => {
-        const toCreateObject: any = {
-          ...doc,
-        };
-        delete toCreateObject.$databaseId;
-        delete toCreateObject.$collectionId;
-        delete toCreateObject.$createdAt;
-        delete toCreateObject.$updatedAt;
-        delete toCreateObject.$id;
-        delete toCreateObject.$permissions;
-        return tryAwaitWithRetry(
-          async () =>
-            await db.createDocument(
-              toDbId,
-              toCollId,
-              doc.$id,
-              toCreateObject,
-              doc.$permissions
-            )
-        );
-      });
-      await Promise.all(batchedPromises);
-      totalDocumentsTransferred += fromCollDocs.documents.length;
-    }
-  }
-
-  MessageFormatter.success(
-    `Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`,
-    { prefix: "Transfer" }
-  );
-};
+export const transferDocumentsBetweenDbsLocalToLocal = async (
+  db: Databases | DatabaseAdapter,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  // Use adapter path when available for bulk operations
+  if (!isLegacyDatabases(db)) {
+    const adapter = db as DatabaseAdapter;
+
+    const pageSize = 1000;
+    let lastId: string | undefined;
+    let totalTransferred = 0;
+
+    while (true) {
+      const queries = [Query.limit(pageSize)];
+      if (lastId) queries.push(Query.cursorAfter(lastId));
+
+      const result = await adapter.listRows({ databaseId: fromDbId, tableId: fromCollId, queries });
+      const rows: any[] = (result as any).rows || (result as any).documents || [];
+      if (!rows.length) break;
+
+      // Prepare rows: strip system fields, keep $id and $permissions
+      const prepared = rows.map((doc) => {
+        const data: any = { ...doc };
+        delete data.$databaseId;
+        delete data.$collectionId;
+        delete data.$createdAt;
+        delete data.$updatedAt;
+        return data; // keep $id and $permissions for upsert
+      });
+
+      // Prefer bulk upsert, then bulk create, then individual
+      if (typeof (adapter as any).bulkUpsertRows === 'function' && adapter.supportsBulkOperations()) {
+        await (adapter as any).bulkUpsertRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
+      } else if (typeof (adapter as any).bulkCreateRows === 'function' && adapter.supportsBulkOperations()) {
+        await (adapter as any).bulkCreateRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
+      } else {
+        for (const row of prepared) {
+          const id = row.$id || ID.unique();
+          const permissions = row.$permissions || [];
+          const { $id, $permissions, ...data } = row;
+          await adapter.createRow({ databaseId: toDbId, tableId: toCollId, id, data, permissions });
+        }
+      }
+
+      totalTransferred += rows.length;
+      if (rows.length < pageSize) break;
+      lastId = rows[rows.length - 1].$id;
+    }
+
+    MessageFormatter.success(
+      `Transferred ${totalTransferred} rows from ${fromDbId}/${fromCollId} to ${toDbId}/${toCollId}`,
+      { prefix: "Transfer" }
+    );
+    return;
+  }
+
+  // Legacy path (Databases) – keep existing behavior
+  const legacyDb = db as Databases;
+  let fromCollDocs = await tryAwaitWithRetry(async () =>
+    legacyDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
+  );
+  let totalDocumentsTransferred = 0;
+
+  if (fromCollDocs.documents.length === 0) {
+    MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
+    return;
+  } else if (fromCollDocs.documents.length < 50) {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: any = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(
+        async () =>
+          await legacyDb.createDocument(
+            toDbId,
+            toCollId,
+            doc.$id,
+            toCreateObject,
+            doc.$permissions
+          )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+  } else {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: any = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(async () =>
+        legacyDb.createDocument(
+          toDbId,
+          toCollId,
+          doc.$id,
+          toCreateObject,
+          doc.$permissions
+        )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+    while (fromCollDocs.documents.length === 50) {
+      fromCollDocs = await tryAwaitWithRetry(
+        async () =>
+          await legacyDb.listDocuments(fromDbId, fromCollId, [
+            Query.limit(50),
+            Query.cursorAfter(
+              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
+            ),
+          ])
+      );
+      const batchedPromises = fromCollDocs.documents.map((doc) => {
+        const toCreateObject: any = {
+          ...doc,
+        };
+        delete toCreateObject.$databaseId;
+        delete toCreateObject.$collectionId;
+        delete toCreateObject.$createdAt;
+        delete toCreateObject.$updatedAt;
+        delete toCreateObject.$id;
+        delete toCreateObject.$permissions;
+        return tryAwaitWithRetry(
+          async () =>
+            await legacyDb.createDocument(
+              toDbId,
+              toCollId,
+              doc.$id,
+              toCreateObject,
+              doc.$permissions
+            )
+        );
+      });
+      await Promise.all(batchedPromises);
+      totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+  }
+
+  MessageFormatter.success(
+    `Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`,
+    { prefix: "Transfer" }
+  );
+};
 
 /**
  * Enhanced document transfer with fault tolerance and exponential backoff
@@ -436,27 +493,24 @@ const transferDocumentBatchWithRetry = async (
   return await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, documents, batchSize);
 };
 
-export const transferDocumentsBetweenDbsLocalToRemote = async (
-  localDb: Databases,
-  endpoint: string,
-  projectId: string,
-  apiKey: string,
-  fromDbId: string,
-  toDbId: string,
-  fromCollId: string,
-  toCollId: string
-) => {
-  MessageFormatter.info(`Starting enhanced document transfer from ${fromCollId} to ${toCollId}...`, { prefix: "Transfer" });
-
-  const client = new Client()
-    .setEndpoint(endpoint)
-    .setProject(projectId)
-    .setKey(apiKey);
-
-  const remoteDb = new Databases(client);
-  let totalDocumentsProcessed = 0;
-  let totalSuccessful = 0;
-  let totalFailed = 0;
+export const transferDocumentsBetweenDbsLocalToRemote = async (
+  localDb: Databases | DatabaseAdapter,
+  endpoint: string,
+  projectId: string,
+  apiKey: string,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  MessageFormatter.info(`Starting enhanced document transfer from ${fromCollId} to ${toCollId}...`, { prefix: "Transfer" });
+
+  // Prefer adapter for remote to enable bulk operations
+  const { adapter: remoteAdapter, client } = await getAdapter(endpoint, projectId, apiKey, 'auto');
+  const remoteDb = new Databases(client); // Legacy fallback for HTTP/individual
+  let totalDocumentsProcessed = 0;
+  let totalSuccessful = 0;
+  let totalFailed = 0;
 
   // Fetch documents in larger batches (1000 at a time)
   let hasMoreDocuments = true;
@@ -468,9 +522,15 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
       queries.push(Query.cursorAfter(lastDocumentId));
     }
 
-    const fromCollDocs = await tryAwaitWithRetry(async () =>
-      localDb.listDocuments(fromDbId, fromCollId, queries)
-    );
+    const fromCollDocs = await tryAwaitWithRetry(async () => {
+      if (isLegacyDatabases(localDb)) {
+        return localDb.listDocuments(fromDbId, fromCollId, queries);
+      } else {
+        const res = await (localDb as DatabaseAdapter).listRows({ databaseId: fromDbId, tableId: fromCollId, queries });
+        const rows = (res as any).rows || (res as any).documents || [];
+        return { documents: rows } as any;
+      }
+    });
 
     if (fromCollDocs.documents.length === 0) {
       hasMoreDocuments = false;
@@ -479,13 +539,27 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
 
     MessageFormatter.progress(`Fetched ${fromCollDocs.documents.length} documents, processing for transfer...`, { prefix: "Transfer" });
 
-    const { successful, failed } = await transferDocumentBatchWithRetry(
-      remoteDb,
-      client,
-      toDbId,
-      toCollId,
-      fromCollDocs.documents
-    );
+    // Prefer remote adapter bulk upsert if available
+    const prepared = fromCollDocs.documents.map((doc: any) => {
+      const data: any = { ...doc };
+      delete data.$databaseId; delete data.$collectionId; delete data.$createdAt; delete data.$updatedAt;
+      return data; // Keep $id and $permissions for upsert
+    });
+
+    let successful = 0; let failed = 0;
+    if (typeof (remoteAdapter as any).bulkUpsertRows === 'function' && remoteAdapter.supportsBulkOperations()) {
+      try {
+        await (remoteAdapter as any).bulkUpsertRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
+        successful = prepared.length;
+      } catch (e) {
+        MessageFormatter.warning('Remote adapter bulk upsert failed, falling back to HTTP/individual', { prefix: 'Transfer' });
+      }
+    }
+
+    if (successful === 0) {
+      const res = await transferDocumentBatchWithRetry(remoteDb, client, toDbId, toCollId, fromCollDocs.documents);
+      successful = res.successful; failed = res.failed;
+    }
 
     totalDocumentsProcessed += fromCollDocs.documents.length;
     totalSuccessful += successful;
@@ -513,4 +587,4 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
   } else {
     MessageFormatter.success(message, { prefix: "Transfer" });
   }
-};
+};

package/src/config/services/ConfigLoaderService.ts

@@ -1,5 +1,6 @@
 import fs from "fs";
 import path from "path";
+import { resolve as resolvePath, dirname, isAbsolute } from "node:path";
 import yaml from "js-yaml";
 import { register } from "tsx/esm/api";
 import { pathToFileURL } from "node:url";
@@ -19,6 +20,7 @@ import {
   type CollectionDiscoveryResult,
   type TableDiscoveryResult,
 } from "../../utils/configDiscovery.js";
+import { expandTildePath } from "../../functions/pathResolution.js";
 
 /**
  * Options for loading collections or tables
@@ -51,6 +53,35 @@ export interface CollectionLoadOptions {
  * - Validates and normalizes configuration data
  */
 export class ConfigLoaderService {
+  /**
+   * Normalizes function dirPath to absolute path
+   * @param func Function configuration object
+   * @param configDir Directory containing the config file
+   * @returns Function with normalized dirPath
+   */
+  private normalizeFunctionPath(func: any, configDir: string): any {
+    if (!func.dirPath) {
+      return func;
+    }
+
+    // Expand tilde first
+    const expandedPath = expandTildePath(func.dirPath);
+
+    // If already absolute, return as-is
+    if (isAbsolute(expandedPath)) {
+      return {
+        ...func,
+        dirPath: expandedPath
+      };
+    }
+
+    // Resolve relative to config directory
+    return {
+      ...func,
+      dirPath: resolvePath(configDir, expandedPath)
+    };
+  }
+
   /**
    * Loads configuration from a discovered path, auto-detecting the type
    * @param configPath Path to the configuration file
@@ -78,6 +109,13 @@ export class ConfigLoaderService {
         );
       }
 
+      const configDir = path.dirname(configPath);
+
+      // Normalize function paths
+      const normalizedFunctions = partialConfig.functions
+        ? partialConfig.functions.map(func => this.normalizeFunctionPath(func, configDir))
+        : [];
+
       return {
         appwriteEndpoint: partialConfig.appwriteEndpoint,
         appwriteProject: partialConfig.appwriteProject,
@@ -105,7 +143,7 @@ export class ConfigLoaderService {
         },
         databases: partialConfig.databases || [],
         buckets: partialConfig.buckets || [],
-        functions: partialConfig.functions || [],
+        functions: normalizedFunctions,
         collections: partialConfig.collections || [],
         sessionCookie: partialConfig.sessionCookie,
         authMethod: partialConfig.authMethod || "auto",
@@ -155,6 +193,13 @@ export class ConfigLoaderService {
 
       // Load collections and tables from their respective directories
       const configDir = path.dirname(yamlPath);
+
+      // Normalize function paths
+      if (config.functions) {
+        config.functions = config.functions.map(func =>
+          this.normalizeFunctionPath(func, configDir)
+        );
+      }
       const collectionsDir = path.join(configDir, config.schemaConfig?.collectionsDirectory || "collections");
       const tablesDir = path.join(configDir, config.schemaConfig?.tablesDirectory || "tables");
 
@@ -248,6 +293,14 @@ export class ConfigLoaderService {
         throw new Error(`Failed to load TypeScript config from: ${tsPath}`);
       }
 
+      // Normalize function paths
+      const configDir = path.dirname(tsPath);
+      if (config.functions) {
+        config.functions = config.functions.map(func =>
+          this.normalizeFunctionPath(func, configDir)
+        );
+      }
+
       MessageFormatter.success(`Loaded TypeScript config from: ${tsPath}`, {
         prefix: "Config",
       });
@@ -292,6 +345,14 @@ export class ConfigLoaderService {
         appwriteConfig.collections = collections;
       }
 
+      // Normalize function paths
+      const configDir = path.dirname(jsonPath);
+      if (appwriteConfig.functions) {
+        appwriteConfig.functions = appwriteConfig.functions.map(func =>
+          this.normalizeFunctionPath(func, configDir)
+        );
+      }
+
       MessageFormatter.success(`Loaded project config from: ${jsonPath}`, {
         prefix: "Config",
       });

package/src/examples/yamlTerminologyExample.ts

@@ -21,7 +21,10 @@ import {
 import { createYamlLoader } from "../utils/yamlLoader.js";
 import { YamlImportIntegration } from "../migrations/yaml/YamlImportIntegration.js";
 import { createImportSchemas } from "../migrations/yaml/generateImportSchemas.js";
-import type { CollectionCreate } from "appwrite-utils";
+import {
+  CollectionCreateSchema,
+  type CollectionCreate
+} from "appwrite-utils";
 import fs from "fs";
 import path from "path";
 
@@ -271,7 +274,7 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
 
   try {
     // Example collection for demonstrations
-    const exampleCollection: CollectionCreate = {
+    const exampleCollectionInput: CollectionCreate = {
       name: "Product",
       $id: "product",
       enabled: true,
@@ -286,7 +289,7 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
         },
         {
           key: "price",
-          type: "float",
+          type: "double",
           required: true,
           min: 0
         },
@@ -296,7 +299,8 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
           relationType: "manyToOne",
           relatedCollection: "Categories",
           twoWay: false,
-          onDelete: "setNull"
+          onDelete: "setNull",
+          required: false
         }
       ],
       indexes: [
@@ -308,6 +312,7 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
       ],
       importDefs: []
     };
+    const exampleCollection = CollectionCreateSchema.parse(exampleCollectionInput);
 
     // Run examples
     await generateTemplateExamples(outputDir);
@@ -338,4 +343,4 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
   }
 }
 
-// Note: Functions are already exported above with their declarations
+// Note: Functions are already exported above with their declarations

package/src/functions/deployments.ts

@@ -16,30 +16,7 @@ import {
 } from "./methods.js";
 import ignore from "ignore";
 import { MessageFormatter } from "../shared/messageFormatter.js";
-
-const findFunctionDirectory = (
-  basePath: string,
-  functionName: string
-): string | undefined => {
-  const normalizedName = functionName.toLowerCase().replace(/\s+/g, "-");
-  const dirs = fs.readdirSync(basePath, { withFileTypes: true });
-
-  for (const dir of dirs) {
-    if (dir.isDirectory()) {
-      const fullPath = join(basePath, dir.name);
-      if (dir.name.toLowerCase() === normalizedName) {
-        return fullPath;
-      }
-
-      const nestedResult = findFunctionDirectory(fullPath, functionName);
-      if (nestedResult) {
-        return nestedResult;
-      }
-    }
-  }
-
-  return undefined;
-};
+import { resolveFunctionDirectory, validateFunctionDirectory } from './pathResolution.js';
 
 export const deployFunction = async (
   client: Client,
@@ -183,18 +160,16 @@ export const deployLocalFunction = async (
     functionExists = false;
   }
 
-  const resolvedPath =
-    functionPath ||
-    functionConfig.dirPath ||
-    findFunctionDirectory(process.cwd(), functionName) ||
-    join(
-      process.cwd(),
-      "functions",
-      functionName.toLowerCase().replace(/\s+/g, "-")
-    );
+  const configDirPath = process.cwd(); // TODO: This should be passed from caller
+  const resolvedPath = resolveFunctionDirectory(
+    functionName,
+    configDirPath,
+    functionConfig.dirPath,
+    functionPath
+  );
 
-  if (!fs.existsSync(resolvedPath)) {
-    throw new Error(`Function directory not found at ${resolvedPath}`);
+  if (!validateFunctionDirectory(resolvedPath)) {
+    throw new Error(`Function directory is invalid or missing required files: ${resolvedPath}`);
   }
 
   if (functionConfig.predeployCommands?.length) {