@njdamstra/appwrite-utils-cli 1.8.9

package/src/collections/transferOperations.ts

@@ -0,0 +1,590 @@
+import {
+  Client,
+  Databases,
+  ID,
+  Query,
+} from "node-appwrite";
+import { tryAwaitWithRetry, delay, calculateExponentialBackoff } from "../utils/helperFunctions.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+import { chunk } from "es-toolkit";
+import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
+import { isLegacyDatabases } from "../utils/typeGuards.js";
+import { getAdapter } from "../utils/getClientFromConfig.js";
+
+/**
+ * Transfers all documents from one collection to another in a different database
+ * within the same Appwrite Project
+ */
+export const transferDocumentsBetweenDbsLocalToLocal = async (
+  db: Databases | DatabaseAdapter,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  // Use adapter path when available for bulk operations
+  if (!isLegacyDatabases(db)) {
+    const adapter = db as DatabaseAdapter;
+
+    const pageSize = 1000;
+    let lastId: string | undefined;
+    let totalTransferred = 0;
+
+    while (true) {
+      const queries = [Query.limit(pageSize)];
+      if (lastId) queries.push(Query.cursorAfter(lastId));
+
+      const result = await adapter.listRows({ databaseId: fromDbId, tableId: fromCollId, queries });
+      const rows: any[] = (result as any).rows || (result as any).documents || [];
+      if (!rows.length) break;
+
+      // Prepare rows: strip system fields, keep $id and $permissions
+      const prepared = rows.map((doc) => {
+        const data: any = { ...doc };
+        delete data.$databaseId;
+        delete data.$collectionId;
+        delete data.$createdAt;
+        delete data.$updatedAt;
+        return data; // keep $id and $permissions for upsert
+      });
+
+      // Prefer bulk upsert, then bulk create, then individual
+      if (typeof (adapter as any).bulkUpsertRows === 'function' && adapter.supportsBulkOperations()) {
+        await (adapter as any).bulkUpsertRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
+      } else if (typeof (adapter as any).bulkCreateRows === 'function' && adapter.supportsBulkOperations()) {
+        await (adapter as any).bulkCreateRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
+      } else {
+        for (const row of prepared) {
+          const id = row.$id || ID.unique();
+          const permissions = row.$permissions || [];
+          const { $id, $permissions, ...data } = row;
+          await adapter.createRow({ databaseId: toDbId, tableId: toCollId, id, data, permissions });
+        }
+      }
+
+      totalTransferred += rows.length;
+      if (rows.length < pageSize) break;
+      lastId = rows[rows.length - 1].$id;
+    }
+
+    MessageFormatter.success(
+      `Transferred ${totalTransferred} rows from ${fromDbId}/${fromCollId} to ${toDbId}/${toCollId}`,
+      { prefix: "Transfer" }
+    );
+    return;
+  }
+
+  // Legacy path (Databases) – keep existing behavior
+  const legacyDb = db as Databases;
+  let fromCollDocs = await tryAwaitWithRetry(async () =>
+    legacyDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
+  );
+  let totalDocumentsTransferred = 0;
+
+  if (fromCollDocs.documents.length === 0) {
+    MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
+    return;
+  } else if (fromCollDocs.documents.length < 50) {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: any = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(
+        async () =>
+          await legacyDb.createDocument(
+            toDbId,
+            toCollId,
+            doc.$id,
+            toCreateObject,
+            doc.$permissions
+          )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+  } else {
+    const batchedPromises = fromCollDocs.documents.map((doc) => {
+      const toCreateObject: any = {
+        ...doc,
+      };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      return tryAwaitWithRetry(async () =>
+        legacyDb.createDocument(
+          toDbId,
+          toCollId,
+          doc.$id,
+          toCreateObject,
+          doc.$permissions
+        )
+      );
+    });
+    await Promise.all(batchedPromises);
+    totalDocumentsTransferred += fromCollDocs.documents.length;
+    while (fromCollDocs.documents.length === 50) {
+      fromCollDocs = await tryAwaitWithRetry(
+        async () =>
+          await legacyDb.listDocuments(fromDbId, fromCollId, [
+            Query.limit(50),
+            Query.cursorAfter(
+              fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
+            ),
+          ])
+      );
+      const batchedPromises = fromCollDocs.documents.map((doc) => {
+        const toCreateObject: any = {
+          ...doc,
+        };
+        delete toCreateObject.$databaseId;
+        delete toCreateObject.$collectionId;
+        delete toCreateObject.$createdAt;
+        delete toCreateObject.$updatedAt;
+        delete toCreateObject.$id;
+        delete toCreateObject.$permissions;
+        return tryAwaitWithRetry(
+          async () =>
+            await legacyDb.createDocument(
+              toDbId,
+              toCollId,
+              doc.$id,
+              toCreateObject,
+              doc.$permissions
+            )
+        );
+      });
+      await Promise.all(batchedPromises);
+      totalDocumentsTransferred += fromCollDocs.documents.length;
+    }
+  }
+
+  MessageFormatter.success(
+    `Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`,
+    { prefix: "Transfer" }
+  );
+};
+
+/**
+ * Enhanced document transfer with fault tolerance and exponential backoff
+ */
+const transferDocumentWithRetry = async (
+  db: Databases,
+  dbId: string,
+  collectionId: string,
+  documentId: string,
+  documentData: any,
+  permissions: string[],
+  maxRetries: number = 3,
+  retryCount: number = 0
+): Promise<boolean> => {
+  try {
+    await db.createDocument(
+      dbId,
+      collectionId,
+      documentId,
+      documentData,
+      permissions
+    );
+    return true;
+  } catch (error: any) {
+    // Check if document already exists
+    if (error.code === 409 || error.message?.toLowerCase().includes('already exists')) {
+      await db.updateDocument(
+        dbId,
+        collectionId,
+        documentId,
+        documentData,
+        permissions
+      );
+    }
+
+    if (retryCount < maxRetries) {
+      // Calculate exponential backoff: 1s, 2s, 4s, max 8s
+      const exponentialDelay = calculateExponentialBackoff(retryCount, 1000, 8000);
+      MessageFormatter.progress(`Retrying document ${documentId} (attempt ${retryCount + 1}/${maxRetries}, backoff: ${exponentialDelay}ms)`, { prefix: "Transfer" });
+
+      await delay(exponentialDelay);
+
+      return await transferDocumentWithRetry(
+        db,
+        dbId,
+        collectionId,
+        documentId,
+        documentData,
+        permissions,
+        maxRetries,
+        retryCount + 1
+      );
+    }
+
+    MessageFormatter.error(`Failed to transfer document ${documentId} after ${maxRetries} retries`, error, { prefix: "Transfer" });
+    return false;
+  }
+};
+
+/**
+ * Check if endpoint supports bulk operations (cloud.appwrite.io)
+ */
+const supportsBulkOperations = (endpoint: string): boolean => {
+  return endpoint.includes('cloud.appwrite.io');
+};
+
+/**
+ * Direct HTTP implementation of bulk upsert API
+ */
+const bulkUpsertDocuments = async (
+  client: any,
+  dbId: string,
+  collectionId: string,
+  documents: any[]
+): Promise<any> => {
+  const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+  const url = new URL(client.config.endpoint + apiPath);
+
+  const headers = {
+    'Content-Type': 'application/json',
+    'X-Appwrite-Project': client.config.project,
+    'X-Appwrite-Key': client.config.key
+  };
+
+  const response = await fetch(url.toString(), {
+    method: 'PUT',
+    headers,
+    body: JSON.stringify({ documents })
+  });
+
+  if (!response.ok) {
+    const errorData: any = await response.json().catch(() => ({ message: 'Unknown error' }));
+    throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+  }
+
+  return await response.json();
+};
+
+/**
+ * Direct HTTP implementation of bulk create API
+ */
+const bulkCreateDocuments = async (
+  client: any,
+  dbId: string,
+  collectionId: string,
+  documents: any[]
+): Promise<any> => {
+  const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+  const url = new URL(client.config.endpoint + apiPath);
+
+  const headers = {
+    'Content-Type': 'application/json',
+    'X-Appwrite-Project': client.config.project,
+    'X-Appwrite-Key': client.config.key
+  };
+
+  const response = await fetch(url.toString(), {
+    method: 'POST',
+    headers,
+    body: JSON.stringify({ documents })
+  });
+
+  if (!response.ok) {
+    const errorData: any = await response.json().catch(() => ({ message: 'Unknown error' }));
+    throw new Error(`Bulk create failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+  }
+
+  return await response.json();
+};
+
+/**
+ * Enhanced bulk document creation using direct HTTP calls
+ */
+const transferDocumentsBulkUpsert = async (
+  client: any,
+  dbId: string,
+  collectionId: string,
+  documents: any[],
+  maxBatchSize: number = 1000
+): Promise<{ successful: number; failed: number }> => {
+  let successful = 0;
+  let failed = 0;
+
+  // Prepare documents for bulk upsert
+  const preparedDocs = documents.map(doc => {
+    const toCreateObject: any = { ...doc };
+    delete toCreateObject.$databaseId;
+    delete toCreateObject.$collectionId;
+    delete toCreateObject.$createdAt;
+    delete toCreateObject.$updatedAt;
+
+    // Keep $id and $permissions for upsert functionality
+    return toCreateObject;
+  });
+
+  // Process in batches based on plan limits
+  const documentBatches = chunk(preparedDocs, maxBatchSize);
+
+  for (const batch of documentBatches) {
+    MessageFormatter.progress(`Bulk upserting ${batch.length} documents...`, { prefix: "Transfer" });
+
+    try {
+      // Try bulk upsert with direct HTTP call
+      const result = await bulkUpsertDocuments(client, dbId, collectionId, batch);
+      successful += result.documents?.length || batch.length;
+      MessageFormatter.success(`Bulk upserted ${result.documents?.length || batch.length} documents`, { prefix: "Transfer" });
+
+    } catch (error: any) {
+      MessageFormatter.progress(`Bulk upsert failed, trying smaller batch size...`, { prefix: "Transfer" });
+
+      // If bulk upsert fails, try with smaller batch size (Pro plan limit)
+      if (maxBatchSize > 100) {
+        const smallerBatches = chunk(batch, 100);
+
+        for (const smallBatch of smallerBatches) {
+          try {
+            const result = await bulkUpsertDocuments(client, dbId, collectionId, smallBatch);
+            successful += result.documents?.length || smallBatch.length;
+            MessageFormatter.success(`Bulk upserted ${result.documents?.length || smallBatch.length} documents (smaller batch)`, { prefix: "Transfer" });
+          } catch (smallBatchError: any) {
+            MessageFormatter.progress(`Smaller batch failed, falling back to individual transfers...`, { prefix: "Transfer" });
+
+            // Fall back to individual document transfer for this batch
+            const db = new Databases(client);
+            const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(
+              db, dbId, collectionId, smallBatch.map((doc, index) => ({
+                ...doc,
+                $id: documents[documentBatches.indexOf(batch) * maxBatchSize + smallerBatches.indexOf(smallBatch) * 100 + index]?.$id || ID.unique(),
+                $permissions: documents[documentBatches.indexOf(batch) * maxBatchSize + smallerBatches.indexOf(smallBatch) * 100 + index]?.$permissions || []
+              }))
+            );
+            successful += indivSuccessful;
+            failed += indivFailed;
+          }
+
+          // Add delay between batches
+          await delay(200);
+        }
+      } else {
+        // Fall back to individual document transfer
+        const db = new Databases(client);
+        const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(
+          db, dbId, collectionId, batch.map((doc, index) => ({
+            ...doc,
+            $id: documents[documentBatches.indexOf(batch) * maxBatchSize + index]?.$id || ID.unique(),
+            $permissions: documents[documentBatches.indexOf(batch) * maxBatchSize + index]?.$permissions || []
+          }))
+        );
+        successful += indivSuccessful;
+        failed += indivFailed;
+      }
+    }
+
+    // Add delay between major batches
+    if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
+      await delay(500);
+    }
+  }
+
+  return { successful, failed };
+};
+
+/**
+ * Fallback batch document transfer with individual retry logic
+ */
+const transferDocumentBatchWithRetryFallback = async (
+  db: Databases,
+  dbId: string,
+  collectionId: string,
+  documents: any[],
+  batchSize: number = 10
+): Promise<{ successful: number; failed: number }> => {
+  let successful = 0;
+  let failed = 0;
+
+  // Process documents in smaller batches to avoid overwhelming the server
+  const documentBatches = chunk(documents, batchSize);
+
+  for (const batch of documentBatches) {
+    MessageFormatter.progress(`Processing batch of ${batch.length} documents...`, { prefix: "Transfer" });
+
+    const batchPromises = batch.map(async (doc) => {
+      const toCreateObject: Partial<typeof doc> = { ...doc };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+
+      const result = await transferDocumentWithRetry(
+        db,
+        dbId,
+        collectionId,
+        doc.$id,
+        toCreateObject,
+        doc.$permissions || []
+      );
+
+      return { docId: doc.$id, success: result };
+    });
+
+    const results = await Promise.allSettled(batchPromises);
+
+    results.forEach((result, index) => {
+      if (result.status === 'fulfilled') {
+        if (result.value.success) {
+          successful++;
+        } else {
+          failed++;
+        }
+      } else {
+        MessageFormatter.error(`Batch promise rejected for document ${batch[index].$id}`, new Error(String(result.reason)), { prefix: "Transfer" });
+        failed++;
+      }
+    });
+
+    // Add delay between batches to avoid rate limiting
+    if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
+      await delay(500);
+    }
+  }
+
+  return { successful, failed };
+};
+
+/**
+ * Enhanced batch document transfer with fault tolerance and bulk API support
+ */
+const transferDocumentBatchWithRetry = async (
+  db: Databases,
+  client: any,
+  dbId: string,
+  collectionId: string,
+  documents: any[],
+  batchSize: number = 10
+): Promise<{ successful: number; failed: number }> => {
+  // Check if we can use bulk operations
+  if (supportsBulkOperations(client.config.endpoint)) {
+    MessageFormatter.info(`Using bulk upsert API for faster document transfer`, { prefix: "Transfer" });
+
+    // Try with Scale plan limit first (2500), then Pro (1000), then Free (100)
+    const batchSizes = [1000, 100]; // Start with Pro plan, fallback to Free
+
+    for (const maxBatchSize of batchSizes) {
+      try {
+        return await transferDocumentsBulkUpsert(client, dbId, collectionId, documents, maxBatchSize);
+      } catch (error: any) {
+        MessageFormatter.progress(`Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`, { prefix: "Transfer" });
+        continue;
+      }
+    }
+
+    // If all bulk operations fail, fall back to individual transfers
+    MessageFormatter.progress(`All bulk operations failed, falling back to individual document transfers`, { prefix: "Transfer" });
+  }
+
+  // Fall back to individual document transfer
+  return await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, documents, batchSize);
+};
+
+export const transferDocumentsBetweenDbsLocalToRemote = async (
+  localDb: Databases | DatabaseAdapter,
+  endpoint: string,
+  projectId: string,
+  apiKey: string,
+  fromDbId: string,
+  toDbId: string,
+  fromCollId: string,
+  toCollId: string
+) => {
+  MessageFormatter.info(`Starting enhanced document transfer from ${fromCollId} to ${toCollId}...`, { prefix: "Transfer" });
+
+  // Prefer adapter for remote to enable bulk operations
+  const { adapter: remoteAdapter, client } = await getAdapter(endpoint, projectId, apiKey, 'auto');
+  const remoteDb = new Databases(client); // Legacy fallback for HTTP/individual
+  let totalDocumentsProcessed = 0;
+  let totalSuccessful = 0;
+  let totalFailed = 0;
+
+  // Fetch documents in larger batches (1000 at a time)
+  let hasMoreDocuments = true;
+  let lastDocumentId: string | undefined;
+
+  while (hasMoreDocuments) {
+    const queries = [Query.limit(1000)]; // Fetch 1000 documents at a time
+    if (lastDocumentId) {
+      queries.push(Query.cursorAfter(lastDocumentId));
+    }
+
+    const fromCollDocs = await tryAwaitWithRetry(async () => {
+      if (isLegacyDatabases(localDb)) {
+        return localDb.listDocuments(fromDbId, fromCollId, queries);
+      } else {
+        const res = await (localDb as DatabaseAdapter).listRows({ databaseId: fromDbId, tableId: fromCollId, queries });
+        const rows = (res as any).rows || (res as any).documents || [];
+        return { documents: rows } as any;
+      }
+    });
+
+    if (fromCollDocs.documents.length === 0) {
+      hasMoreDocuments = false;
+      break;
+    }
+
+    MessageFormatter.progress(`Fetched ${fromCollDocs.documents.length} documents, processing for transfer...`, { prefix: "Transfer" });
+
+    // Prefer remote adapter bulk upsert if available
+    const prepared = fromCollDocs.documents.map((doc: any) => {
+      const data: any = { ...doc };
+      delete data.$databaseId; delete data.$collectionId; delete data.$createdAt; delete data.$updatedAt;
+      return data; // Keep $id and $permissions for upsert
+    });
+
+    let successful = 0; let failed = 0;
+    if (typeof (remoteAdapter as any).bulkUpsertRows === 'function' && remoteAdapter.supportsBulkOperations()) {
+      try {
+        await (remoteAdapter as any).bulkUpsertRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
+        successful = prepared.length;
+      } catch (e) {
+        MessageFormatter.warning('Remote adapter bulk upsert failed, falling back to HTTP/individual', { prefix: 'Transfer' });
+      }
+    }
+
+    if (successful === 0) {
+      const res = await transferDocumentBatchWithRetry(remoteDb, client, toDbId, toCollId, fromCollDocs.documents);
+      successful = res.successful; failed = res.failed;
+    }
+
+    totalDocumentsProcessed += fromCollDocs.documents.length;
+    totalSuccessful += successful;
+    totalFailed += failed;
+
+    // Check if we have more documents to process
+    if (fromCollDocs.documents.length < 1000) {
+      hasMoreDocuments = false;
+    } else {
+      lastDocumentId = fromCollDocs.documents[fromCollDocs.documents.length - 1].$id;
+    }
+
+    MessageFormatter.debug(`Batch complete: ${successful} successful, ${failed} failed`, undefined, { prefix: "Transfer" });
+  }
+
+  if (totalDocumentsProcessed === 0) {
+    MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
+    return;
+  }
+
+  const message = `Total documents processed: ${totalDocumentsProcessed}, successful: ${totalSuccessful}, failed: ${totalFailed}`;
+
+  if (totalFailed > 0) {
+    MessageFormatter.warning(message, { prefix: "Transfer" });
+  } else {
+    MessageFormatter.success(message, { prefix: "Transfer" });
+  }
+};