appwrite-utils-cli 1.7.9 → 1.8.2

package/src/collections/transferOperations.ts

@@ -1,120 +1,177 @@
-import {
-  Client,
-  Databases,
-  ID,
-  Query,
-} from "node-appwrite";
-import { tryAwaitWithRetry, delay, calculateExponentialBackoff } from "../utils/helperFunctions.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
-import { chunk } from "es-toolkit";
+import {
+  Client,
+  Databases,
+  ID,
+  Query,
+} from "node-appwrite";
+import { tryAwaitWithRetry, delay, calculateExponentialBackoff } from "../utils/helperFunctions.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+import { chunk } from "es-toolkit";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
+import { isLegacyDatabases } from "../utils/typeGuards.js";
+import { getAdapter } from "../utils/getClientFromConfig.js";
 
 /**
  * Transfers all documents from one collection to another in a different database
  * within the same Appwrite Project
  */
-export const transferDocumentsBetweenDbsLocalToLocal = async (
-  db: Databases,
-  fromDbId: string,
-  toDbId: string,
-  fromCollId: string,
-  toCollId: string
-) => {
-  let fromCollDocs = await tryAwaitWithRetry(async () =>
-    db.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
-  );
-  let totalDocumentsTransferred = 0;
-
-  if (fromCollDocs.documents.length === 0) {
-    MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
-    return;
-  } else if (fromCollDocs.documents.length < 50) {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
-      const toCreateObject: any = {
-        ...doc,
-      };
-      delete toCreateObject.$databaseId;
-      delete toCreateObject.$collectionId;
-      delete toCreateObject.$createdAt;
-      delete toCreateObject.$updatedAt;
-      delete toCreateObject.$id;
-      delete toCreateObject.$permissions;
-      return tryAwaitWithRetry(
-        async () =>
-          await db.createDocument(
-            toDbId,
-            toCollId,
-            doc.$id,
-            toCreateObject,
-            doc.$permissions
-          )
-      );
-    });
-    await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
-  } else {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
-      const toCreateObject: any = {
-        ...doc,
-      };
-      delete toCreateObject.$databaseId;
-      delete toCreateObject.$collectionId;
-      delete toCreateObject.$createdAt;
-      delete toCreateObject.$updatedAt;
-      delete toCreateObject.$id;
-      delete toCreateObject.$permissions;
-      return tryAwaitWithRetry(async () =>
-        db.createDocument(
-          toDbId,
-          toCollId,
-          doc.$id,
-          toCreateObject,
-          doc.$permissions
-        )
-      );
-    });
-    await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
-    while (fromCollDocs.documents.length === 50) {
-      fromCollDocs = await tryAwaitWithRetry(
-        async () =>
-          await db.listDocuments(fromDbId, fromCollId, [
-            Query.limit(50),
-            Query.cursorAfter(
-              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
-            ),
-          ])
-      );
-      const batchedPromises = fromCollDocs.documents.map((doc) => {
-        const toCreateObject: any = {
-          ...doc,
-        };
-        delete toCreateObject.$databaseId;
-        delete toCreateObject.$collectionId;
-        delete toCreateObject.$createdAt;
-        delete toCreateObject.$updatedAt;
-        delete toCreateObject.$id;
-        delete toCreateObject.$permissions;
-        return tryAwaitWithRetry(
-          async () =>
-            await db.createDocument(
-              toDbId,
-              toCollId,
-              doc.$id,
-              toCreateObject,
-              doc.$permissions
-            )
-        );
-      });
-      await Promise.all(batchedPromises);
-      totalDocumentsTransferred += fromCollDocs.documents.length;
-    }
-  }
-
-  MessageFormatter.success(
-    `Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`,
-    { prefix: "Transfer" }
-  );
-};
+export const transferDocumentsBetweenDbsLocalToLocal = async (
+  db: Databases | DatabaseAdapter,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  // Use adapter path when available for bulk operations
+  if (!isLegacyDatabases(db)) {
+    const adapter = db as DatabaseAdapter;
+
+    const pageSize = 1000;
+    let lastId: string | undefined;
+    let totalTransferred = 0;
+
+    while (true) {
+      const queries = [Query.limit(pageSize)];
+      if (lastId) queries.push(Query.cursorAfter(lastId));
+
+      const result = await adapter.listRows({ databaseId: fromDbId, tableId: fromCollId, queries });
+      const rows: any[] = (result as any).rows || (result as any).documents || [];
+      if (!rows.length) break;
+
+      // Prepare rows: strip system fields, keep $id and $permissions
+      const prepared = rows.map((doc) => {
+        const data: any = { ...doc };
+        delete data.$databaseId;
+        delete data.$collectionId;
+        delete data.$createdAt;
+        delete data.$updatedAt;
+        return data; // keep $id and $permissions for upsert
+      });
+
+      // Prefer bulk upsert, then bulk create, then individual
+      if (typeof (adapter as any).bulkUpsertRows === 'function' && adapter.supportsBulkOperations()) {
+        await (adapter as any).bulkUpsertRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
+      } else if (typeof (adapter as any).bulkCreateRows === 'function' && adapter.supportsBulkOperations()) {
+        await (adapter as any).bulkCreateRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
+      } else {
+        for (const row of prepared) {
+          const id = row.$id || ID.unique();
+          const permissions = row.$permissions || [];
+          const { $id, $permissions, ...data } = row;
+          await adapter.createRow({ databaseId: toDbId, tableId: toCollId, id, data, permissions });
+        }
+      }
+
+      totalTransferred += rows.length;
+      if (rows.length < pageSize) break;
+      lastId = rows[rows.length - 1].$id;
+    }
+
+    MessageFormatter.success(
+      `Transferred ${totalTransferred} rows from ${fromDbId}/${fromCollId} to ${toDbId}/${toCollId}`,
+      { prefix: "Transfer" }
+    );
+    return;
+  }
+
+  // Legacy path (Databases) – keep existing behavior
+  const legacyDb = db as Databases;
+  let fromCollDocs = await tryAwaitWithRetry(async () =>
+    legacyDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
+  );
+  let totalDocumentsTransferred = 0;
+
+  if (fromCollDocs.documents.length === 0) {
+    MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
+    return;
+  } else if (fromCollDocs.documents.length < 50) {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: any = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(
+        async () =>
+          await legacyDb.createDocument(
+            toDbId,
+            toCollId,
+            doc.$id,
+            toCreateObject,
+            doc.$permissions
+          )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+  } else {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: any = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(async () =>
+        legacyDb.createDocument(
+          toDbId,
+          toCollId,
+          doc.$id,
+          toCreateObject,
+          doc.$permissions
+        )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+    while (fromCollDocs.documents.length === 50) {
+      fromCollDocs = await tryAwaitWithRetry(
+        async () =>
+          await legacyDb.listDocuments(fromDbId, fromCollId, [
+            Query.limit(50),
+            Query.cursorAfter(
+              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
+            ),
+          ])
+      );
+      const batchedPromises = fromCollDocs.documents.map((doc) => {
+        const toCreateObject: any = {
+          ...doc,
+        };
+        delete toCreateObject.$databaseId;
+        delete toCreateObject.$collectionId;
+        delete toCreateObject.$createdAt;
+        delete toCreateObject.$updatedAt;
+        delete toCreateObject.$id;
+        delete toCreateObject.$permissions;
+        return tryAwaitWithRetry(
+          async () =>
+            await legacyDb.createDocument(
+              toDbId,
+              toCollId,
+              doc.$id,
+              toCreateObject,
+              doc.$permissions
+            )
+        );
+      });
+      await Promise.all(batchedPromises);
+      totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+  }
+
+  MessageFormatter.success(
+    `Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`,
+    { prefix: "Transfer" }
+  );
+};
 
 /**
  * Enhanced document transfer with fault tolerance and exponential backoff
@@ -436,27 +493,24 @@ const transferDocumentBatchWithRetry = async (
   return await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, documents, batchSize);
 };
 
-export const transferDocumentsBetweenDbsLocalToRemote = async (
-  localDb: Databases,
-  endpoint: string,
-  projectId: string,
-  apiKey: string,
-  fromDbId: string,
-  toDbId: string,
-  fromCollId: string,
-  toCollId: string
-) => {
-  MessageFormatter.info(`Starting enhanced document transfer from ${fromCollId} to ${toCollId}...`, { prefix: "Transfer" });
-
-  const client = new Client()
-    .setEndpoint(endpoint)
-    .setProject(projectId)
-    .setKey(apiKey);
-
-  const remoteDb = new Databases(client);
-  let totalDocumentsProcessed = 0;
-  let totalSuccessful = 0;
-  let totalFailed = 0;
+export const transferDocumentsBetweenDbsLocalToRemote = async (
+  localDb: Databases | DatabaseAdapter,
+  endpoint: string,
+  projectId: string,
+  apiKey: string,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  MessageFormatter.info(`Starting enhanced document transfer from ${fromCollId} to ${toCollId}...`, { prefix: "Transfer" });
+
+  // Prefer adapter for remote to enable bulk operations
+  const { adapter: remoteAdapter, client } = await getAdapter(endpoint, projectId, apiKey, 'auto');
+  const remoteDb = new Databases(client); // Legacy fallback for HTTP/individual
+  let totalDocumentsProcessed = 0;
+  let totalSuccessful = 0;
+  let totalFailed = 0;
 
   // Fetch documents in larger batches (1000 at a time)
   let hasMoreDocuments = true;
@@ -468,9 +522,15 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
       queries.push(Query.cursorAfter(lastDocumentId));
     }
 
-    const fromCollDocs = await tryAwaitWithRetry(async () =>
-      localDb.listDocuments(fromDbId, fromCollId, queries)
-    );
+    const fromCollDocs = await tryAwaitWithRetry(async () => {
+      if (isLegacyDatabases(localDb)) {
+        return localDb.listDocuments(fromDbId, fromCollId, queries);
+      } else {
+        const res = await (localDb as DatabaseAdapter).listRows({ databaseId: fromDbId, tableId: fromCollId, queries });
+        const rows = (res as any).rows || (res as any).documents || [];
+        return { documents: rows } as any;
+      }
+    });
 
     if (fromCollDocs.documents.length === 0) {
       hasMoreDocuments = false;
@@ -479,13 +539,27 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
 
     MessageFormatter.progress(`Fetched ${fromCollDocs.documents.length} documents, processing for transfer...`, { prefix: "Transfer" });
 
-    const { successful, failed } = await transferDocumentBatchWithRetry(
-      remoteDb,
-      client,
-      toDbId,
-      toCollId,
-      fromCollDocs.documents
-    );
+    // Prefer remote adapter bulk upsert if available
+    const prepared = fromCollDocs.documents.map((doc: any) => {
+      const data: any = { ...doc };
+      delete data.$databaseId; delete data.$collectionId; delete data.$createdAt; delete data.$updatedAt;
+      return data; // Keep $id and $permissions for upsert
+    });
+
+    let successful = 0; let failed = 0;
+    if (typeof (remoteAdapter as any).bulkUpsertRows === 'function' && remoteAdapter.supportsBulkOperations()) {
+      try {
+        await (remoteAdapter as any).bulkUpsertRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
+        successful = prepared.length;
+      } catch (e) {
+        MessageFormatter.warning('Remote adapter bulk upsert failed, falling back to HTTP/individual', { prefix: 'Transfer' });
+      }
+    }
+
+    if (successful === 0) {
+      const res = await transferDocumentBatchWithRetry(remoteDb, client, toDbId, toCollId, fromCollDocs.documents);
+      successful = res.successful; failed = res.failed;
+    }
 
     totalDocumentsProcessed += fromCollDocs.documents.length;
     totalSuccessful += successful;
@@ -513,4 +587,4 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
   } else {
     MessageFormatter.success(message, { prefix: "Transfer" });
   }
-};
+};

package/src/examples/yamlTerminologyExample.ts

@@ -21,7 +21,10 @@ import {
 import { createYamlLoader } from "../utils/yamlLoader.js";
 import { YamlImportIntegration } from "../migrations/yaml/YamlImportIntegration.js";
 import { createImportSchemas } from "../migrations/yaml/generateImportSchemas.js";
-import type { CollectionCreate } from "appwrite-utils";
+import {
+  CollectionCreateSchema,
+  type CollectionCreate
+} from "appwrite-utils";
 import fs from "fs";
 import path from "path";
 
@@ -271,7 +274,7 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
 
   try {
     // Example collection for demonstrations
-    const exampleCollection: CollectionCreate = {
+    const exampleCollectionInput: CollectionCreate = {
       name: "Product",
       $id: "product",
       enabled: true,
@@ -286,7 +289,7 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
         },
         {
           key: "price",
-          type: "float",
+          type: "double",
           required: true,
           min: 0
         },
@@ -296,7 +299,8 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
           relationType: "manyToOne",
           relatedCollection: "Categories",
           twoWay: false,
-          onDelete: "setNull"
+          onDelete: "setNull",
+          required: false
         }
       ],
       indexes: [
@@ -308,6 +312,7 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
       ],
       importDefs: []
     };
+    const exampleCollection = CollectionCreateSchema.parse(exampleCollectionInput);
 
     // Run examples
     await generateTemplateExamples(outputDir);
@@ -338,4 +343,4 @@ export async function runYamlTerminologyExamples(outputDir: string): Promise<voi
   }
 }
 
-// Note: Functions are already exported above with their declarations
+// Note: Functions are already exported above with their declarations

package/src/functions/fnConfigDiscovery.ts

@@ -0,0 +1,103 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import yaml from 'js-yaml';
+import { homedir } from 'node:os';
+import { AppwriteFunctionSchema, type AppwriteFunction } from 'appwrite-utils';
+import { shouldIgnoreDirectory } from '../utils/directoryUtils.js';
+import { MessageFormatter } from '../shared/messageFormatter.js';
+
+function findGitRoot(startDir: string): string {
+  let dir = path.resolve(startDir);
+  while (dir !== path.parse(dir).root) {
+    if (fs.existsSync(path.join(dir, '.git'))) return dir;
+    const parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return path.resolve(startDir);
+}
+
+function expandTilde(p: string): string {
+  if (!p) return p;
+  if (p === '~' || p.startsWith('~/')) return p.replace(/^~(?=$|\/|\\)/, homedir());
+  return p;
+}
+
+export function discoverFnConfigs(startDir: string): AppwriteFunction[] {
+  const root = findGitRoot(startDir);
+  const results: AppwriteFunction[] = [];
+
+  const visit = (dir: string, depth = 0) => {
+    if (depth > 5) return; // cap depth
+    const base = path.basename(dir);
+    if (shouldIgnoreDirectory(base)) return;
+    let entries: fs.Dirent[] = [];
+    try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+
+    // Check for .fnconfig.yaml / .fnconfig.yml
+    for (const fname of ['.fnconfig.yaml', '.fnconfig.yml']) {
+      const cfgPath = path.join(dir, fname);
+      if (fs.existsSync(cfgPath)) {
+        try {
+          const raw = fs.readFileSync(cfgPath, 'utf8');
+          const data = yaml.load(raw) as any;
+          const parsed = AppwriteFunctionSchema.parse({
+            $id: data.id || data.$id,
+            name: data.name,
+            runtime: data.runtime,
+            execute: data.execute || [],
+            events: data.events || [],
+            schedule: data.schedule,
+            timeout: data.timeout,
+            enabled: data.enabled,
+            logging: data.logging,
+            entrypoint: data.entrypoint,
+            commands: data.commands,
+            scopes: data.scopes,
+            installationId: data.installationId,
+            providerRepositoryId: data.providerRepositoryId,
+            providerBranch: data.providerBranch,
+            providerSilentMode: data.providerSilentMode,
+            providerRootDirectory: data.providerRootDirectory,
+            templateRepository: data.templateRepository,
+            templateOwner: data.templateOwner,
+            templateRootDirectory: data.templateRootDirectory,
+            templateVersion: data.templateVersion,
+            specification: data.specification,
+            dirPath: data.dirPath,
+            predeployCommands: data.predeployCommands,
+            deployDir: data.deployDir,
+            ignore: data.ignore,
+          });
+
+          // Resolve dirPath relative to the config file directory
+          let dirPath = parsed.dirPath || '.';
+          dirPath = expandTilde(dirPath);
+          if (!path.isAbsolute(dirPath)) dirPath = path.resolve(path.dirname(cfgPath), dirPath);
+          const merged: AppwriteFunction = { ...parsed, dirPath };
+          results.push(merged);
+        } catch (e) {
+          MessageFormatter.warning(`Failed to parse ${cfgPath}: ${e instanceof Error ? e.message : String(e)}`, { prefix: 'Functions' });
+        }
+      }
+    }
+
+    for (const entry of entries) {
+      if (entry.isDirectory()) visit(path.join(dir, entry.name), depth + 1);
+    }
+  };
+
+  visit(root, 0);
+  return results;
+}
+
+export function mergeDiscoveredFunctions(
+  central: AppwriteFunction[] = [],
+  discovered: AppwriteFunction[] = []
+): AppwriteFunction[] {
+  const map = new Map<string, AppwriteFunction>();
+  for (const f of central) if (f?.$id) map.set(f.$id, f);
+  for (const f of discovered) if (f?.$id) map.set(f.$id, f); // discovered overrides
+  return Array.from(map.values());
+}
+