appwrite-utils-cli 0.0.3 → 0.0.4

package/src/migrations/relationships.ts

@@ -5,22 +5,25 @@ import type {
   Attribute,
   RelationshipAttribute,
 } from "./schema.js";
+import { logger } from "./logging.js";
 
+/**
+ * Finds collections that have defined relationship attributes.
+ */
 export const findCollectionsWithRelationships = (config: AppwriteConfig) => {
-  const toReturn = new Map<string, Attribute[]>();
-  // Map of collection name to array of attributes so we can update the relationships
+  const toReturn = new Map<string, RelationshipAttribute[]>();
   for (const collection of config.collections) {
     if (collection.attributes) {
       for (const attribute of collection.attributes) {
-        if (attribute.type === "relationship") {
-          if (!toReturn.has(collection.name)) {
-            toReturn.set(collection.name, []);
-          }
-          toReturn.get(collection.name)?.push(attribute);
-          if (!toReturn.has(attribute.relatedCollection)) {
-            toReturn.set(attribute.relatedCollection, []);
-          }
-          toReturn.get(attribute.relatedCollection)?.push(attribute);
+        if (
+          attribute.type === "relationship" &&
+          attribute.twoWay &&
+          attribute.side === "parent"
+        ) {
+          toReturn.set(collection.name, toReturn.get(collection.name) || []);
+          toReturn
+            .get(collection.name)
+            ?.push(attribute as RelationshipAttribute);
         }
       }
     }
@@ -28,33 +31,6 @@ export const findCollectionsWithRelationships = (config: AppwriteConfig) => {
   return toReturn;
 };
 
-async function fetchAllDocuments(
-  dbId: string,
-  database: Databases,
-  collectionId: string
-): Promise<Models.Document[]> {
-  let allDocuments: Models.Document[] = [];
-  let after; // This will be used for pagination
-
-  while (true) {
-    const response: Models.DocumentList<Models.Document> =
-      await database.listDocuments(dbId, collectionId, [
-        Query.limit(100), // Adjust based on the maximum limit your database allows
-        ...(after ? [Query.cursorAfter(after)] : []),
-      ]);
-
-    allDocuments = allDocuments.concat(response.documents);
-
-    if (response.documents.length === 0 || response.total === 0) {
-      break; // Exit the loop if there are no more documents to fetch
-    }
-
-    after = response.documents[response.documents.length - 1].$id; // Prepare for the next page
-  }
-
-  return allDocuments;
-}
-
 export async function resolveAndUpdateRelationships(
   dbId: string,
   database: Databases,
@@ -92,36 +68,50 @@ async function processCollection(
   collection: Models.Collection,
   relAttributeMap: RelationshipAttribute[]
 ) {
-  const allDocuments = await fetchAllDocuments(dbId, database, collection.$id);
-  console.log(
-    `Fetched ${allDocuments.length} documents from collection: ${collection.name}`
-  );
-
-  const batchSize = 10; // Process documents in batches of 10
+  let after; // For pagination
+  let hasMore = true;
 
-  for (let i = 0; i < allDocuments.length; i += batchSize) {
-    const batch = allDocuments.slice(i, i + batchSize);
+  while (hasMore) {
+    const response: Models.DocumentList<Models.Document> =
+      await database.listDocuments(dbId, collection.$id, [
+        Query.limit(100), // Fetch documents in batches of 100
+        ...(after ? [Query.cursorAfter(after)] : []),
+      ]);
 
-    const updates = await prepareDocumentUpdates(
-      database,
-      dbId,
-      collection.name,
-      batch,
-      relAttributeMap
+    const documents = response.documents;
+    console.log(
+      `Fetched ${documents.length} documents from collection: ${collection.name}`
     );
 
-    // Execute updates for the current batch
-    await executeUpdatesInBatches(dbId, database, updates);
+    if (documents.length > 0) {
+      const updates = await prepareDocumentUpdates(
+        database,
+        dbId,
+        collection.name,
+        documents,
+        relAttributeMap
+      );
+
+      // Execute updates for the current batch
+      await executeUpdatesInBatches(dbId, database, updates);
+    }
+
+    if (documents.length === 100) {
+      after = documents[documents.length - 1].$id; // Prepare for the next page
+    } else {
+      hasMore = false; // No more documents to fetch
+    }
   }
 }
 
 async function findDocumentsByOriginalId(
   database: Databases,
   dbId: string,
-  relatedCollectionId: string,
+  targetCollection: Models.Collection,
   targetKey: string,
   originalId: string | string[]
 ): Promise<Models.Document[] | undefined> {
+  const relatedCollectionId = targetCollection.$id;
   const collection = await database.listCollections(dbId, [
     Query.equal("$id", relatedCollectionId),
   ]);
@@ -150,11 +140,10 @@ async function findDocumentsByOriginalId(
     ...queries,
     Query.limit(500), // Adjust the limit based on your needs or implement pagination
   ]);
-  if (response.total > 0) {
-    return undefined;
-  }
 
-  if (response.documents.length > 0) {
+  if (response.documents.length < 0) {
+    return undefined;
+  } else if (response.documents.length > 0) {
     return response.documents;
   } else {
     return undefined;
@@ -185,85 +174,81 @@ async function prepareDocumentUpdates(
     return [];
   }
 
-  // Function to process a batch of documents
-  const processDocumentBatch = async (docBatch: Models.Document[]) => {
-    for (const doc of docBatch) {
-      let updatePayload: { [key: string]: any } = {};
+  for (const doc of documents) {
+    let updatePayload: { [key: string]: any } = {};
 
-      for (const rel of relationships) {
-        // Check if the relationship has importMapping defined
-        if (!rel.importMapping) {
-          continue;
-        }
-        // Skip if not dealing with the parent side of a two-way relationship
-        if (rel.twoWay && rel.side !== "parent") {
-          console.log(
-            "Not processing child side of two-way relationship, skipping..."
-          );
-          continue;
-        }
-        const isSingleReference =
-          rel.relationType === "oneToOne" || rel.relationType === "manyToOne";
-        const originalIdField = rel.importMapping.originalIdField;
-        const targetField = rel.importMapping.targetField || originalIdField; // Use originalIdField if targetField is not specified
-        const originalId = doc[originalIdField as keyof typeof doc];
-        if (!originalId) {
-          continue; // Skip if the document doesn't have the original ID field
-        }
-        const collection = await database.listCollections(dbId, [
+    for (const rel of relationships) {
+      // Skip if not dealing with the parent side of a two-way relationship
+      if (rel.twoWay && rel.side !== "parent") {
+        console.log("Skipping non-parent side of two-way relationship...");
+        continue;
+      }
+
+      const isSingleReference =
+        rel.relationType === "oneToOne" || rel.relationType === "manyToOne";
+      const originalIdField = rel.importMapping?.originalIdField;
+      const targetField = rel.importMapping?.targetField || originalIdField; // Use originalIdField if targetField is not specified
+      if (!originalIdField) {
+        console.log("Missing originalIdField in importMapping, skipping...");
+        continue;
+      }
+      const originalId = doc[originalIdField as keyof typeof doc];
+      if (!originalId) {
+        continue;
+      }
+
+      const relatedCollection = (
+        await database.listCollections(dbId, [
           Query.equal("name", rel.relatedCollection),
-        ]);
-        if (collection.total === 0) {
-          console.log(
-            `Collection ${rel.relatedCollection} doesn't exist, skipping...`
-          );
-          continue; // Skip if the related collection doesn't exist
-        }
-        const relatedCollectionId = collection.collections[0].$id;
-
-        // Find documents in the related collection that match the original ID
-        const foundDocuments = await findDocumentsByOriginalId(
-          database,
-          dbId,
-          relatedCollectionId,
-          targetField,
-          originalId
+        ])
+      ).collections[0];
+
+      if (!relatedCollection) {
+        console.log(
+          `Related collection ${rel.relatedCollection} not found, skipping...`
         );
+        continue;
+      }
 
-        if (foundDocuments && foundDocuments.length > 0) {
-          const relationshipKey = rel.key;
-          const existingRefs = doc[relationshipKey as keyof typeof doc] || [];
-          let existingRefIds: string[] = [];
-          if (Array.isArray(existingRefs)) {
-            // @ts-ignore
-            existingRefIds = existingRefs.map((ref) => ref.$id);
-          } else if (existingRefs) {
-            // @ts-ignore
-            existingRefIds = [existingRefs.$id];
-          }
-
-          const newRefs = foundDocuments.map((fd) => fd.$id);
-          const allRefs = [...new Set([...existingRefIds, ...newRefs])]; // Combine and remove duplicates
-
-          // Update logic based on the relationship cardinality
-          updatePayload[relationshipKey] = isSingleReference
-            ? newRefs[0] || existingRefIds[0]
-            : allRefs;
+      const foundDocuments = await findDocumentsByOriginalId(
+        database,
+        dbId,
+        relatedCollection,
+        targetField!,
+        originalId
+      );
+
+      if (foundDocuments && foundDocuments.length > 0) {
+        const relationshipKey = rel.key;
+        const existingRefs = doc[relationshipKey as keyof typeof doc] || [];
+        let existingRefIds: string[] = [];
+        if (Array.isArray(existingRefs)) {
+          // @ts-ignore
+          existingRefIds = existingRefs.map((ref) => ref.$id);
+        } else if (existingRefs) {
+          // @ts-ignore
+          existingRefIds = [existingRefs.$id];
         }
-      }
 
-      if (Object.keys(updatePayload).length > 0) {
-        updates.push({
-          collectionId: thisCollectionId,
-          documentId: doc.$id,
-          updatePayload: updatePayload,
-        });
+        const newRefs = foundDocuments.map((fd) => fd.$id);
+        const allRefs = [...new Set([...existingRefIds, ...newRefs])]; // Combine and remove duplicates
+
+        // Update logic based on the relationship cardinality
+        updatePayload[relationshipKey] = isSingleReference
+          ? newRefs[0] || existingRefIds[0]
+          : allRefs;
+        console.log(`Updating ${relationshipKey} with ${allRefs.length} refs`);
       }
     }
-  };
 
-  // Process documents in batches
-  await processInBatches(documents, 25, processDocumentBatch);
+    if (Object.keys(updatePayload).length > 0) {
+      updates.push({
+        collectionId: thisCollectionId,
+        documentId: doc.$id,
+        updatePayload: updatePayload,
+      });
+    }
+  }
 
   return updates;
 }
@@ -320,14 +305,14 @@ async function executeUpdatesInBatches(
             update.updatePayload
           )
           .catch((error) => {
-            console.error("Error updating document: ", error);
-            console.error(
-              "Document ID: ",
-              update.documentId,
-              "Collection ID: ",
-              update.collectionId,
-              "Document update payload: ",
-              JSON.stringify(update.updatePayload, undefined, 4)
+            logger.error(
+              `Error updating doc ${
+                update.documentId
+              } in ${dbId}, update payload: ${JSON.stringify(
+                update.updatePayload,
+                undefined,
+                4
+              )}, error: ${error}`
             );
           })
       )

package/src/migrations/schema.ts

@@ -378,6 +378,63 @@ export const indexSchema = z.object({
 
 export type Index = z.infer<typeof indexSchema>;
 
+export const AttributeMappingsSchema = z.array(
+  z.object({
+    oldKey: z
+      .string()
+      .optional()
+      .describe("The key of the attribute in the old document"),
+    oldKeys: z
+      .array(z.string())
+      .optional()
+      .describe(
+        "The keys of the attribute in the old document, if there are more than one"
+      ),
+    targetKey: z
+      .string()
+      .describe("The key of the attribute in the new document"),
+    fileData: z
+      .object({
+        name: z
+          .string()
+          .describe("The name of the file, can use template strings"),
+        path: z
+          .string()
+          .describe("The path of the file, relative to the appwrite folder"),
+      })
+      .optional()
+      .describe(
+        "The file data to use for the import, if defined it will upload and replace with ID"
+      ),
+    converters: z
+      .array(z.string())
+      .describe("The converters to use for the import")
+      .default([]),
+    validationActions: z
+      .array(
+        z.object({
+          action: z.string(),
+          params: z.array(z.string().startsWith("{").endsWith("}")),
+        })
+      )
+      .describe(
+        "The after import actions and parameter placeholders (they'll be replaced with the actual data) to use for the import"
+      )
+      .default([]),
+    postImportActions: z
+      .array(
+        z.object({
+          action: z.string(),
+          params: z.array(z.string().or(z.record(z.string(), z.any()))),
+        })
+      )
+      .describe(
+        "The after import actions and parameter placeholders (they'll be replaced with the actual data) to use for the import"
+      )
+      .default([]),
+  })
+);
+
 export const collectionSchema = z.object({
   $id: z
     .string()
@@ -450,63 +507,8 @@ export const collectionSchema = z.object({
           .describe(
             "Configuration for mapping and resolving the update during data import"
           ),
-        attributeMappings: z.array(
-          z.object({
-            oldKey: z
-              .string()
-              .optional()
-              .describe("The key of the attribute in the old document"),
-            oldKeys: z
-              .array(z.string())
-              .optional()
-              .describe(
-                "The keys of the attribute in the old document, if there are more than one"
-              ),
-            targetKey: z
-              .string()
-              .describe("The key of the attribute in the new document"),
-            fileData: z
-              .object({
-                name: z
-                  .string()
-                  .describe("The name of the file, can use template strings"),
-                path: z
-                  .string()
-                  .describe(
-                    "The path of the file, relative to the appwrite folder"
-                  ),
-              })
-              .optional()
-              .describe(
-                "The file data to use for the import, if defined it will upload and replace with ID"
-              ),
-            converters: z
-              .array(z.string())
-              .describe("The converters to use for the import")
-              .default([]),
-            validationActions: z
-              .array(
-                z.object({
-                  action: z.string(),
-                  params: z.array(z.string().startsWith("{").endsWith("}")),
-                })
-              )
-              .describe(
-                "The after import actions and parameter placeholders (they'll be replaced with the actual data) to use for the import"
-              )
-              .default([]),
-            postImportActions: z
-              .array(
-                z.object({
-                  action: z.string(),
-                  params: z.array(z.string().or(z.record(z.string(), z.any()))),
-                })
-              )
-              .describe(
-                "The after import actions and parameter placeholders (they'll be replaced with the actual data) to use for the import"
-              )
-              .default([]),
-          })
+        attributeMappings: AttributeMappingsSchema.describe(
+          "The attribute mappings to use for the import"
         ),
       })
     )
@@ -586,5 +588,5 @@ export type ConfigDatabases = AppwriteConfig["databases"];
 export type ConfigDatabase = ConfigDatabases[number];
 export type ImportDefs = ConfigCollections[number]["importDefs"];
 export type ImportDef = ImportDefs[number];
-export type AttributeMappings = ImportDefs[number]["attributeMappings"];
+export type AttributeMappings = z.infer<typeof AttributeMappingsSchema>;
 export type AttributeMapping = AttributeMappings[number];