appwrite-utils-cli 0.0.2 → 0.0.4

package/dist/migrations/relationships.js

@@ -1,42 +1,27 @@
 import { Databases, Query } from "node-appwrite";
 import { fetchAllCollections } from "./collections.js";
+import { logger } from "./logging.js";
+/**
+ * Finds collections that have defined relationship attributes.
+ */
 export const findCollectionsWithRelationships = (config) => {
     const toReturn = new Map();
-    // Map of collection name to array of attributes so we can update the relationships
     for (const collection of config.collections) {
         if (collection.attributes) {
             for (const attribute of collection.attributes) {
-                if (attribute.type === "relationship") {
-                    if (!toReturn.has(collection.name)) {
-                        toReturn.set(collection.name, []);
-                    }
-                    toReturn.get(collection.name)?.push(attribute);
-                    if (!toReturn.has(attribute.relatedCollection)) {
-                        toReturn.set(attribute.relatedCollection, []);
-                    }
-                    toReturn.get(attribute.relatedCollection)?.push(attribute);
+                if (attribute.type === "relationship" &&
+                    attribute.twoWay &&
+                    attribute.side === "parent") {
+                    toReturn.set(collection.name, toReturn.get(collection.name) || []);
+                    toReturn
+                        .get(collection.name)
+                        ?.push(attribute);
                 }
             }
         }
     }
     return toReturn;
 };
-async function fetchAllDocuments(dbId, database, collectionId) {
-    let allDocuments = [];
-    let after; // This will be used for pagination
-    while (true) {
-        const response = await database.listDocuments(dbId, collectionId, [
-            Query.limit(100), // Adjust based on the maximum limit your database allows
-            ...(after ? [Query.cursorAfter(after)] : []),
-        ]);
-        allDocuments = allDocuments.concat(response.documents);
-        if (response.documents.length === 0 || response.total === 0) {
-            break; // Exit the loop if there are no more documents to fetch
-        }
-        after = response.documents[response.documents.length - 1].$id; // Prepare for the next page
-    }
-    return allDocuments;
-}
 export async function resolveAndUpdateRelationships(dbId, database, config) {
     const collections = await fetchAllCollections(dbId, database);
     const collectionsWithRelationships = findCollectionsWithRelationships(config);
@@ -53,17 +38,30 @@ export async function resolveAndUpdateRelationships(dbId, database, config) {
     console.log(`Completed relationship resolution and update for database ID: ${dbId}`);
 }
 async function processCollection(dbId, database, collection, relAttributeMap) {
-    const allDocuments = await fetchAllDocuments(dbId, database, collection.$id);
-    console.log(`Fetched ${allDocuments.length} documents from collection: ${collection.name}`);
-    const batchSize = 10; // Process documents in batches of 10
-    for (let i = 0; i < allDocuments.length; i += batchSize) {
-        const batch = allDocuments.slice(i, i + batchSize);
-        const updates = await prepareDocumentUpdates(database, dbId, collection.name, batch, relAttributeMap);
-        // Execute updates for the current batch
-        await executeUpdatesInBatches(dbId, database, updates);
+    let after; // For pagination
+    let hasMore = true;
+    while (hasMore) {
+        const response = await database.listDocuments(dbId, collection.$id, [
+            Query.limit(100), // Fetch documents in batches of 100
+            ...(after ? [Query.cursorAfter(after)] : []),
+        ]);
+        const documents = response.documents;
+        console.log(`Fetched ${documents.length} documents from collection: ${collection.name}`);
+        if (documents.length > 0) {
+            const updates = await prepareDocumentUpdates(database, dbId, collection.name, documents, relAttributeMap);
+            // Execute updates for the current batch
+            await executeUpdatesInBatches(dbId, database, updates);
+        }
+        if (documents.length === 100) {
+            after = documents[documents.length - 1].$id; // Prepare for the next page
+        }
+        else {
+            hasMore = false; // No more documents to fetch
+        }
     }
 }
-async function findDocumentsByOriginalId(database, dbId, relatedCollectionId, targetKey, originalId) {
+async function findDocumentsByOriginalId(database, dbId, targetCollection, targetKey, originalId) {
+    const relatedCollectionId = targetCollection.$id;
     const collection = await database.listCollections(dbId, [
         Query.equal("$id", relatedCollectionId),
     ]);
@@ -90,10 +88,10 @@ async function findDocumentsByOriginalId(database, dbId, relatedCollectionId, ta
         ...queries,
         Query.limit(500), // Adjust the limit based on your needs or implement pagination
     ]);
-    if (response.total > 0) {
+    if (response.documents.length < 0) {
         return undefined;
     }
-    if (response.documents.length > 0) {
+    else if (response.documents.length > 0) {
         return response.documents;
     }
     else {
@@ -109,68 +107,62 @@ async function prepareDocumentUpdates(database, dbId, collectionName, documents,
         console.log(`No collection found with name: ${collectionName}`);
         return [];
     }
-    // Function to process a batch of documents
-    const processDocumentBatch = async (docBatch) => {
-        for (const doc of docBatch) {
-            let updatePayload = {};
-            for (const rel of relationships) {
-                // Check if the relationship has importMapping defined
-                if (!rel.importMapping) {
-                    continue;
-                }
-                // Skip if not dealing with the parent side of a two-way relationship
-                if (rel.twoWay && rel.side !== "parent") {
-                    console.log("Not processing child side of two-way relationship, skipping...");
-                    continue;
-                }
-                const isSingleReference = rel.relationType === "oneToOne" || rel.relationType === "manyToOne";
-                const originalIdField = rel.importMapping.originalIdField;
-                const targetField = rel.importMapping.targetField || originalIdField; // Use originalIdField if targetField is not specified
-                const originalId = doc[originalIdField];
-                if (!originalId) {
-                    continue; // Skip if the document doesn't have the original ID field
-                }
-                const collection = await database.listCollections(dbId, [
-                    Query.equal("name", rel.relatedCollection),
-                ]);
-                if (collection.total === 0) {
-                    console.log(`Collection ${rel.relatedCollection} doesn't exist, skipping...`);
-                    continue; // Skip if the related collection doesn't exist
+    for (const doc of documents) {
+        let updatePayload = {};
+        for (const rel of relationships) {
+            // Skip if not dealing with the parent side of a two-way relationship
+            if (rel.twoWay && rel.side !== "parent") {
+                console.log("Skipping non-parent side of two-way relationship...");
+                continue;
+            }
+            const isSingleReference = rel.relationType === "oneToOne" || rel.relationType === "manyToOne";
+            const originalIdField = rel.importMapping?.originalIdField;
+            const targetField = rel.importMapping?.targetField || originalIdField; // Use originalIdField if targetField is not specified
+            if (!originalIdField) {
+                console.log("Missing originalIdField in importMapping, skipping...");
+                continue;
+            }
+            const originalId = doc[originalIdField];
+            if (!originalId) {
+                continue;
+            }
+            const relatedCollection = (await database.listCollections(dbId, [
+                Query.equal("name", rel.relatedCollection),
+            ])).collections[0];
+            if (!relatedCollection) {
+                console.log(`Related collection ${rel.relatedCollection} not found, skipping...`);
+                continue;
+            }
+            const foundDocuments = await findDocumentsByOriginalId(database, dbId, relatedCollection, targetField, originalId);
+            if (foundDocuments && foundDocuments.length > 0) {
+                const relationshipKey = rel.key;
+                const existingRefs = doc[relationshipKey] || [];
+                let existingRefIds = [];
+                if (Array.isArray(existingRefs)) {
+                    // @ts-ignore
+                    existingRefIds = existingRefs.map((ref) => ref.$id);
                 }
-                const relatedCollectionId = collection.collections[0].$id;
-                // Find documents in the related collection that match the original ID
-                const foundDocuments = await findDocumentsByOriginalId(database, dbId, relatedCollectionId, targetField, originalId);
-                if (foundDocuments && foundDocuments.length > 0) {
-                    const relationshipKey = rel.key;
-                    const existingRefs = doc[relationshipKey] || [];
-                    let existingRefIds = [];
-                    if (Array.isArray(existingRefs)) {
-                        // @ts-ignore
-                        existingRefIds = existingRefs.map((ref) => ref.$id);
-                    }
-                    else if (existingRefs) {
-                        // @ts-ignore
-                        existingRefIds = [existingRefs.$id];
-                    }
-                    const newRefs = foundDocuments.map((fd) => fd.$id);
-                    const allRefs = [...new Set([...existingRefIds, ...newRefs])]; // Combine and remove duplicates
-                    // Update logic based on the relationship cardinality
-                    updatePayload[relationshipKey] = isSingleReference
-                        ? newRefs[0] || existingRefIds[0]
-                        : allRefs;
+                else if (existingRefs) {
+                    // @ts-ignore
+                    existingRefIds = [existingRefs.$id];
                 }
-            }
-            if (Object.keys(updatePayload).length > 0) {
-                updates.push({
-                    collectionId: thisCollectionId,
-                    documentId: doc.$id,
-                    updatePayload: updatePayload,
-                });
+                const newRefs = foundDocuments.map((fd) => fd.$id);
+                const allRefs = [...new Set([...existingRefIds, ...newRefs])]; // Combine and remove duplicates
+                // Update logic based on the relationship cardinality
+                updatePayload[relationshipKey] = isSingleReference
+                    ? newRefs[0] || existingRefIds[0]
+                    : allRefs;
+                console.log(`Updating ${relationshipKey} with ${allRefs.length} refs`);
             }
         }
-    };
-    // Process documents in batches
-    await processInBatches(documents, 25, processDocumentBatch);
+        if (Object.keys(updatePayload).length > 0) {
+            updates.push({
+                collectionId: thisCollectionId,
+                documentId: doc.$id,
+                updatePayload: updatePayload,
+            });
+        }
+    }
     return updates;
 }
 async function processInBatches(items, batchSize, processFunction) {
@@ -202,8 +194,7 @@ async function executeUpdatesInBatches(dbId, database, updates) {
         await Promise.all(batch.map((update) => database
             .updateDocument(dbId, update.collectionId, update.documentId, update.updatePayload)
             .catch((error) => {
-            console.error("Error updating document: ", error);
-            console.error("Document ID: ", update.documentId, "Collection ID: ", update.collectionId, "Document update payload: ", JSON.stringify(update.updatePayload, undefined, 4));
+            logger.error(`Error updating doc ${update.documentId} in ${dbId}, update payload: ${JSON.stringify(update.updatePayload, undefined, 4)}, error: ${error}`);
         })));
     }
 }

package/dist/migrations/schema.d.ts

@@ -676,6 +676,76 @@ export declare const indexSchema: z.ZodObject<{
     orders?: string[] | undefined;
 }>;
 export type Index = z.infer<typeof indexSchema>;
+export declare const AttributeMappingsSchema: z.ZodArray<z.ZodObject<{
+    oldKey: z.ZodOptional<z.ZodString>;
+    oldKeys: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    targetKey: z.ZodString;
+    fileData: z.ZodOptional<z.ZodObject<{
+        name: z.ZodString;
+        path: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        path: string;
+        name: string;
+    }, {
+        path: string;
+        name: string;
+    }>>;
+    converters: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+    validationActions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        action: z.ZodString;
+        params: z.ZodArray<z.ZodString, "many">;
+    }, "strip", z.ZodTypeAny, {
+        params: string[];
+        action: string;
+    }, {
+        params: string[];
+        action: string;
+    }>, "many">>;
+    postImportActions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        action: z.ZodString;
+        params: z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodAny>]>, "many">;
+    }, "strip", z.ZodTypeAny, {
+        params: (string | Record<string, any>)[];
+        action: string;
+    }, {
+        params: (string | Record<string, any>)[];
+        action: string;
+    }>, "many">>;
+}, "strip", z.ZodTypeAny, {
+    targetKey: string;
+    converters: string[];
+    validationActions: {
+        params: string[];
+        action: string;
+    }[];
+    postImportActions: {
+        params: (string | Record<string, any>)[];
+        action: string;
+    }[];
+    oldKey?: string | undefined;
+    oldKeys?: string[] | undefined;
+    fileData?: {
+        path: string;
+        name: string;
+    } | undefined;
+}, {
+    targetKey: string;
+    oldKey?: string | undefined;
+    oldKeys?: string[] | undefined;
+    fileData?: {
+        path: string;
+        name: string;
+    } | undefined;
+    converters?: string[] | undefined;
+    validationActions?: {
+        params: string[];
+        action: string;
+    }[] | undefined;
+    postImportActions?: {
+        params: (string | Record<string, any>)[];
+        action: string;
+    }[] | undefined;
+}>, "many">;
 export declare const collectionSchema: z.ZodObject<{
     $id: z.ZodDefault<z.ZodOptional<z.ZodString>>;
     $createdAt: z.ZodString;
@@ -1206,6 +1276,7 @@ export declare const collectionSchema: z.ZodObject<{
             targetField?: string | undefined;
         } | undefined;
     })[];
+    name: string;
     $id: string;
     $createdAt: string;
     $updatedAt: string;
@@ -1213,7 +1284,6 @@ export declare const collectionSchema: z.ZodObject<{
         permission: string;
         target: string;
     }[];
-    name: string;
     enabled: boolean;
     documentSecurity: boolean;
     indexes: {
@@ -1253,9 +1323,9 @@ export declare const collectionSchema: z.ZodObject<{
     }[];
     databaseId?: string | undefined;
 }, {
+    name: string;
     $createdAt: string;
     $updatedAt: string;
-    name: string;
     $id?: string | undefined;
     $permissions?: {
         permission: string;
@@ -1917,12 +1987,12 @@ export declare const CollectionCreateSchema: z.ZodObject<Omit<{
             targetField?: string | undefined;
         } | undefined;
     })[];
+    name: string;
     $id: string;
     $permissions: {
         permission: string;
         target: string;
     }[];
-    name: string;
     enabled: boolean;
     documentSecurity: boolean;
     indexes: {
@@ -2114,11 +2184,11 @@ export declare const AppwriteConfigSchema: z.ZodObject<{
         $id: z.ZodString;
         name: z.ZodString;
     }, "strip", z.ZodTypeAny, {
-        $id: string;
         name: string;
-    }, {
         $id: string;
+    }, {
         name: string;
+        $id: string;
     }>, "many">>;
     collections: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodObject<Omit<{
         $id: z.ZodDefault<z.ZodOptional<z.ZodString>>;
@@ -2650,12 +2720,12 @@ export declare const AppwriteConfigSchema: z.ZodObject<{
                 targetField?: string | undefined;
             } | undefined;
         })[];
+        name: string;
         $id: string;
         $permissions: {
             permission: string;
             target: string;
         }[];
-        name: string;
         enabled: boolean;
         documentSecurity: boolean;
         indexes: {
@@ -2841,8 +2911,8 @@ export declare const AppwriteConfigSchema: z.ZodObject<{
     documentBucketId: string;
     usersCollectionName: string;
     databases: {
-        $id: string;
         name: string;
+        $id: string;
     }[];
     collections: {
         attributes: ({
@@ -2932,12 +3002,12 @@ export declare const AppwriteConfigSchema: z.ZodObject<{
                 targetField?: string | undefined;
             } | undefined;
         })[];
+        name: string;
         $id: string;
         $permissions: {
             permission: string;
             target: string;
         }[];
-        name: string;
         enabled: boolean;
         documentSecurity: boolean;
         indexes: {
@@ -2993,8 +3063,8 @@ export declare const AppwriteConfigSchema: z.ZodObject<{
     documentBucketId?: string | undefined;
     usersCollectionName?: string | undefined;
     databases?: {
-        $id: string;
         name: string;
+        $id: string;
     }[] | undefined;
     collections?: {
         name: string;
@@ -3137,6 +3207,6 @@ export type ConfigDatabases = AppwriteConfig["databases"];
 export type ConfigDatabase = ConfigDatabases[number];
 export type ImportDefs = ConfigCollections[number]["importDefs"];
 export type ImportDef = ImportDefs[number];
-export type AttributeMappings = ImportDefs[number]["attributeMappings"];
+export type AttributeMappings = z.infer<typeof AttributeMappingsSchema>;
 export type AttributeMapping = AttributeMappings[number];
 export {};

package/dist/migrations/schema.js

@@ -324,6 +324,48 @@ export const indexSchema = z.object({
     attributes: z.array(z.string()),
     orders: z.array(z.string()).optional(),
 });
+export const AttributeMappingsSchema = z.array(z.object({
+    oldKey: z
+        .string()
+        .optional()
+        .describe("The key of the attribute in the old document"),
+    oldKeys: z
+        .array(z.string())
+        .optional()
+        .describe("The keys of the attribute in the old document, if there are more than one"),
+    targetKey: z
+        .string()
+        .describe("The key of the attribute in the new document"),
+    fileData: z
+        .object({
+        name: z
+            .string()
+            .describe("The name of the file, can use template strings"),
+        path: z
+            .string()
+            .describe("The path of the file, relative to the appwrite folder"),
+    })
+        .optional()
+        .describe("The file data to use for the import, if defined it will upload and replace with ID"),
+    converters: z
+        .array(z.string())
+        .describe("The converters to use for the import")
+        .default([]),
+    validationActions: z
+        .array(z.object({
+        action: z.string(),
+        params: z.array(z.string().startsWith("{").endsWith("}")),
+    }))
+        .describe("The after import actions and parameter placeholders (they'll be replaced with the actual data) to use for the import")
+        .default([]),
+    postImportActions: z
+        .array(z.object({
+        action: z.string(),
+        params: z.array(z.string().or(z.record(z.string(), z.any()))),
+    }))
+        .describe("The after import actions and parameter placeholders (they'll be replaced with the actual data) to use for the import")
+        .default([]),
+}));
 export const collectionSchema = z.object({
     $id: z
         .string()
@@ -383,48 +425,7 @@ export const collectionSchema = z.object({
         })
             .optional()
             .describe("Configuration for mapping and resolving the update during data import"),
-        attributeMappings: z.array(z.object({
-            oldKey: z
-                .string()
-                .optional()
-                .describe("The key of the attribute in the old document"),
-            oldKeys: z
-                .array(z.string())
-                .optional()
-                .describe("The keys of the attribute in the old document, if there are more than one"),
-            targetKey: z
-                .string()
-                .describe("The key of the attribute in the new document"),
-            fileData: z
-                .object({
-                name: z
-                    .string()
-                    .describe("The name of the file, can use template strings"),
-                path: z
-                    .string()
-                    .describe("The path of the file, relative to the appwrite folder"),
-            })
-                .optional()
-                .describe("The file data to use for the import, if defined it will upload and replace with ID"),
-            converters: z
-                .array(z.string())
-                .describe("The converters to use for the import")
-                .default([]),
-            validationActions: z
-                .array(z.object({
-                action: z.string(),
-                params: z.array(z.string().startsWith("{").endsWith("}")),
-            }))
-                .describe("The after import actions and parameter placeholders (they'll be replaced with the actual data) to use for the import")
-                .default([]),
-            postImportActions: z
-                .array(z.object({
-                action: z.string(),
-                params: z.array(z.string().or(z.record(z.string(), z.any()))),
-            }))
-                .describe("The after import actions and parameter placeholders (they'll be replaced with the actual data) to use for the import")
-                .default([]),
-        })),
+        attributeMappings: AttributeMappingsSchema.describe("The attribute mappings to use for the import"),
     }))
         .optional()
         .default([])

package/dist/migrations/schemaStrings.js

@@ -99,9 +99,12 @@ export class SchemaGenerator {
             let endNameLazy = `${relatedPascalName}Schema`;
             if (relatedCollection[2] === "array") {
                 endNameTypes += "[]";
-                endNameLazy += ".array()";
+                endNameLazy += ".array().default([])";
+            }
+            else if (!(relatedCollection[2] === "array")) {
+                endNameTypes += " | null";
+                endNameLazy += ".nullish()";
             }
-            endNameLazy += ".nullish()";
             imports += `import { ${relatedPascalName}Schema, type ${relatedPascalName} } from "./${relatedCamelName}";\n`;
             relatedTypes += `${relatedCollection[1]}?: ${endNameTypes};\n`;
             if (relatedTypes.length > 0 && curNum !== maxNum) {

package/dist/schemas/authUser.d.ts

@@ -30,9 +30,9 @@ export declare const AuthUserSchema: z.ZodObject<{
 export type AuthUser = z.infer<typeof AuthUserSchema>;
 export declare const AuthUserCreateSchema: z.ZodObject<{
     email: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+    name: z.ZodOptional<z.ZodNullable<z.ZodString>>;
     $createdAt: z.ZodOptional<z.ZodString>;
     $updatedAt: z.ZodOptional<z.ZodString>;
-    name: z.ZodOptional<z.ZodNullable<z.ZodString>>;
     phone: z.ZodOptional<z.ZodNullable<z.ZodString>>;
     prefs: z.ZodDefault<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>>;
     labels: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
@@ -42,17 +42,17 @@ export declare const AuthUserCreateSchema: z.ZodObject<{
     prefs: Record<string, string>;
     labels: string[];
     email?: string | null | undefined;
+    name?: string | null | undefined;
     $createdAt?: string | undefined;
     $updatedAt?: string | undefined;
-    name?: string | null | undefined;
     phone?: string | null | undefined;
     userId?: string | undefined;
     password?: string | undefined;
 }, {
     email?: string | null | undefined;
+    name?: string | null | undefined;
     $createdAt?: string | undefined;
     $updatedAt?: string | undefined;
-    name?: string | null | undefined;
     phone?: string | null | undefined;
     prefs?: Record<string, string> | undefined;
     labels?: string[] | undefined;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "main": "src/main.ts",
   "type": "module",
   "repository": {