appwrite-utils-cli 0.0.262 → 0.0.263

package/README.md

@@ -15,12 +15,20 @@
 
 ## Installation
 
-To use `appwrite-utils-cli`, you should install it globally via npm to make it accessible from anywhere in your command line:
+To use `appwrite-utils-cli`, you can install it globally via npm to make it accessible from anywhere in your command line:
 
 ```bash
 npm install -g appwrite-utils-cli
 ```
 
+However, due to the nature of the speed at which I am developing this project, I would recommend the following command:
+
+```bash
+npx --package=appwrite-utils-cli@latest appwrite-migrate -- --arg1 --arg2 --arg3
+```
+
+**DO NOT INSTALL THIS LOCALLY INTO YOUR PROJECT, IT IS MEANT TO BE USED AS A COMMAND LINE TOOL ONLY**
+
 ## Usage
 
 After installation, you can access the tool directly from your command line using the provided commands. Here's how you can use the different functionalities:
@@ -30,7 +38,7 @@ After installation, you can access the tool directly from your command line usin
 Set up your Appwrite project with necessary configurations:
 
 ```bash
-appwrite-setup
+npx --package=appwrite-utils-cli@latest appwrite-setup
 ```
 
 To generate an example configuration file:
@@ -39,12 +47,20 @@ To generate an example configuration file:
 appwrite-setup --example
 ```
 
+To synchronize your `appwriteConfig.yaml` with your Appwrite Database, first you must run the setup command and enter your Appwrite instances details in the `projectId`, `endpoint`, and `apiKey`, then run the following
+
+```bash
+npx --package=appwrite-utils-cli@latest appwrite-migrate -- --sync
+```
+
+This will initialize your config and generate schemas for your database using ZOD to `src/appwrite/schemas`
+
 ### Running Migrations and Tasks
 
 Run migration and management tasks with specific flags according to your needs:
 
 ```bash
-appwrite-migrate --args
+npx --package=appwrite-utils-cli@latest appwrite-migrate --args
 ```
 
 Replace `--args` with the appropriate options:
@@ -58,17 +74,19 @@ Replace `--args` with the appropriate options:
 - `--import`: Import data into your databases.
 - `--backup`: Perform a backup of your databases.
 - `--wipe-users` or `--wipeUsers`: Wipe all user data.
+- `--write-data` or `--writeData`: Write converted imported data to file
+- `--sync`: Synchronize your project's config and generate schema for your database
 
 For example, to run migrations in a development environment and import data:
 
 ```bash
-appwrite-migrate --dev --import
+npx --package=appwrite-utils-cli@latest appwrite-migrate --dev --import
 ```
 
 To initialize your project, generate schemas, but not import data:
 
 ```bash
-appwrite-migrate --init
+npx --package=appwrite-utils-cli@latest appwrite-migrate --init
 ```
 
 This setup ensures that developers have robust tools at their fingertips to manage complex Appwrite projects effectively from the command line. I also have added logging automatically for information and errors as the console can be hard to keep up with.
@@ -81,6 +99,7 @@ This setup ensures that developers have robust tools at their fingertips to mana
 
 ### Changelog
 
+- 0.0.254: Added `--sync` to synchronize your Appwrite instance with your local `appwriteConfig.yaml` and generate schemas
 - 0.0.253: Added `--writeData` (or `--write-data`) to command to write the output of the import data to a file called dataLoaderOutput in your root dir
 - 0.0.23: Added batching to user deletion
 - 0.0.22: Converted all import processes except `postImportActions` and Relationship Resolution to the local data import, so it should be much faster.

package/dist/main.js

@@ -4,6 +4,7 @@ const args = process.argv.slice(2);
 async function main() {
     const controller = new UtilsController();
     await controller.init();
+    let sync = false;
     let runProd = false;
     let runStaging = false;
     let runDev = false;
@@ -14,6 +15,9 @@ async function main() {
     let importData = false;
     let wipeDocuments = false;
     let shouldWriteFile = false;
+    if (args.includes("--sync")) {
+        sync = true;
+    }
     if (args.includes("--prod")) {
         runProd = true;
     }
@@ -46,6 +50,7 @@ async function main() {
     }
     if (args.includes("--init")) {
         await controller.run({
+            sync: sync,
             runProd: runProd,
             runStaging: runStaging,
             runDev: runDev,
@@ -62,6 +67,7 @@ async function main() {
     }
     else {
         await controller.run({
+            sync: sync,
             runProd: runProd,
             runStaging: runStaging,
             runDev: runDev,

package/dist/migrations/afterImportActions.d.ts

@@ -1,5 +1,7 @@
-import { type Models } from "node-appwrite";
+import { Databases, Storage, type Models } from "node-appwrite";
 import type { AppwriteConfig } from "./schema.js";
+export declare const getDatabaseFromConfig: (config: AppwriteConfig) => Databases;
+export declare const getStorageFromConfig: (config: AppwriteConfig) => Storage;
 export interface AfterImportActions {
     [key: string]: (config: AppwriteConfig, ...args: any[]) => Promise<any>;
 }

package/dist/migrations/afterImportActions.js

@@ -3,7 +3,7 @@ import path from "path";
 import fs from "fs";
 import os from "os";
 import { logger } from "./logging.js";
-const getDatabaseFromConfig = (config) => {
+export const getDatabaseFromConfig = (config) => {
     if (!config.appwriteClient) {
         config.appwriteClient = new Client()
             .setEndpoint(config.appwriteEndpoint)
@@ -12,7 +12,7 @@ const getDatabaseFromConfig = (config) => {
     }
     return new Databases(config.appwriteClient);
 };
-const getStorageFromConfig = (config) => {
+export const getStorageFromConfig = (config) => {
     if (!config.appwriteClient) {
         config.appwriteClient = new Client()
             .setEndpoint(config.appwriteEndpoint)

package/dist/migrations/appwriteToX.d.ts

@@ -0,0 +1,109 @@
+import { type Models } from "node-appwrite";
+import { type AppwriteConfig } from "./schema.js";
+export declare class AppwriteToX {
+    config: AppwriteConfig;
+    updatedConfig: AppwriteConfig;
+    collToAttributeMap: Map<string, ({
+        key: string;
+        type: "string";
+        error: string;
+        required: boolean;
+        array: boolean;
+        size: number;
+        xdefault?: string | null | undefined;
+        encrypted?: boolean | undefined;
+    } | {
+        key: string;
+        type: "integer";
+        error: string;
+        required: boolean;
+        array: boolean;
+        min?: number | undefined;
+        max?: number | undefined;
+        xdefault?: number | null | undefined;
+    } | {
+        key: string;
+        type: "float";
+        error: string;
+        required: boolean;
+        array: boolean;
+        min?: number | undefined;
+        max?: number | undefined;
+        xdefault?: number | null | undefined;
+    } | {
+        key: string;
+        type: "boolean";
+        error: string;
+        required: boolean;
+        array: boolean;
+        xdefault?: boolean | null | undefined;
+    } | {
+        key: string;
+        type: "datetime";
+        error: string;
+        required: boolean;
+        array: boolean;
+        xdefault?: string | null | undefined;
+    } | {
+        key: string;
+        type: "email";
+        error: string;
+        required: boolean;
+        array: boolean;
+        xdefault?: string | null | undefined;
+    } | {
+        key: string;
+        type: "ip";
+        error: string;
+        required: boolean;
+        array: boolean;
+        xdefault?: string | null | undefined;
+    } | {
+        key: string;
+        type: "url";
+        error: string;
+        required: boolean;
+        array: boolean;
+        xdefault?: string | null | undefined;
+    } | {
+        key: string;
+        type: "enum";
+        error: string;
+        required: boolean;
+        array: boolean;
+        elements: string[];
+        xdefault?: string | null | undefined;
+    } | {
+        key: string;
+        type: "relationship";
+        error: string;
+        required: boolean;
+        relatedCollection: string;
+        relationType: "oneToMany" | "manyToOne" | "oneToOne" | "manyToMany";
+        twoWay: boolean;
+        twoWayKey: string;
+        onDelete: "setNull" | "cascade" | "restrict";
+        side: "parent" | "child";
+        array?: boolean | undefined;
+        importMapping?: {
+            originalIdField: string;
+            targetField?: string | undefined;
+        } | undefined;
+    })[]>;
+    appwriteFolderPath: string;
+    constructor(config: AppwriteConfig, appwriteFolderPath: string);
+    parsePermissionString: (permissionString: string) => {
+        permission: string;
+        target: string;
+    };
+    parsePermissionsArray: (permissions: string[]) => ({
+        permission: string;
+        target: string;
+    } | {
+        permission: string;
+        target: string;
+    })[];
+    updateCollectionConfigAttributes: (collection: Models.Collection) => void;
+    appwriteSync(config: AppwriteConfig): Promise<void>;
+    toSchemas(): Promise<void>;
+}

package/dist/migrations/appwriteToX.js

@@ -0,0 +1,88 @@
+import { SchemaGenerator } from "./schemaStrings.js";
+import { Databases, Query } from "node-appwrite";
+import { fetchAllCollections } from "./collections.js";
+import { fetchAllDatabases } from "./databases.js";
+import { collectionSchema, attributeSchema, AppwriteConfigSchema, permissionsSchema, attributesSchema, indexesSchema, } from "./schema.js";
+import { getDatabaseFromConfig } from "./afterImportActions.js";
+export class AppwriteToX {
+    config;
+    updatedConfig;
+    collToAttributeMap = new Map();
+    appwriteFolderPath;
+    constructor(config, appwriteFolderPath) {
+        this.config = config;
+        this.updatedConfig = config;
+        this.appwriteFolderPath = appwriteFolderPath;
+    }
+    // Function to parse a single permission string
+    parsePermissionString = (permissionString) => {
+        const match = permissionString.match(/^(\w+)\('([^']+)'\)$/);
+        if (!match) {
+            throw new Error(`Invalid permission format: ${permissionString}`);
+        }
+        return {
+            permission: match[1],
+            target: match[2],
+        };
+    };
+    // Function to parse an array of permission strings
+    parsePermissionsArray = (permissions) => {
+        const parsedPermissions = permissionsSchema.parse(permissions);
+        // Validate the parsed permissions using Zod
+        return parsedPermissions ?? [];
+    };
+    updateCollectionConfigAttributes = (collection) => {
+        for (const attribute of collection.attributes) {
+            const attributeMap = this.collToAttributeMap.get(collection.name);
+            const attributeParsed = attributeSchema.parse(attribute);
+            this.collToAttributeMap
+                .get(collection.name)
+                ?.push(attributeParsed);
+        }
+    };
+    async appwriteSync(config) {
+        const db = getDatabaseFromConfig(config);
+        const databases = await fetchAllDatabases(db);
+        let updatedConfig = { ...config };
+        // Loop through each database
+        for (const database of databases) {
+            const collections = await fetchAllCollections(database.$id, db);
+            // Loop through each collection in the current database
+            for (const collection of collections) {
+                const existingCollectionIndex = updatedConfig.collections.findIndex((c) => c.name === collection.name);
+                // Parse the collection permissions and attributes
+                const collPermissions = this.parsePermissionsArray(collection.$permissions);
+                const collAttributes = attributesSchema.parse(collection.attributes);
+                this.collToAttributeMap.set(collection.name, collAttributes);
+                const collIndexes = indexesSchema.parse(collection.indexes);
+                // Prepare the collection object to be added or updated
+                const collToPush = collectionSchema.parse({
+                    name: collection.name,
+                    enabled: collection.enabled,
+                    documentSecurity: collection.documentSecurity,
+                    $createdAt: collection.$createdAt,
+                    $updatedAt: collection.$updatedAt,
+                    $permissions: collPermissions.length > 0 ? collPermissions : undefined,
+                    indexes: collIndexes.length > 0 ? collIndexes : undefined,
+                    attributes: collAttributes.length > 0 ? collAttributes : undefined,
+                });
+                if (existingCollectionIndex !== -1) {
+                    // Update existing collection
+                    updatedConfig.collections[existingCollectionIndex] = collToPush;
+                }
+                else {
+                    // Add new collection
+                    updatedConfig.collections.push(collToPush);
+                }
+            }
+            console.log(`Processed ${collections.length} collections in ${database.name}`);
+        }
+        this.updatedConfig = updatedConfig;
+    }
+    async toSchemas() {
+        await this.appwriteSync(this.config);
+        const generator = new SchemaGenerator(this.updatedConfig, this.appwriteFolderPath);
+        generator.updateYamlSchemas();
+        generator.generateSchemas();
+    }
+}

package/dist/migrations/backup.d.ts

@@ -265,12 +265,12 @@ export declare const getMigrationCollectionSchemas: () => {
             })[];
             name: string;
             $id: string;
+            enabled: boolean;
+            documentSecurity: boolean;
             $permissions: {
                 permission: string;
                 target: string;
             }[];
-            enabled: boolean;
-            documentSecurity: boolean;
             indexes: {
                 key: string;
                 type: "key" | "unique" | "fulltext";
@@ -494,12 +494,12 @@ export declare const getMigrationCollectionSchemas: () => {
             })[];
             name: string;
             $id: string;
+            enabled: boolean;
+            documentSecurity: boolean;
             $permissions: {
                 permission: string;
                 target: string;
             }[];
-            enabled: boolean;
-            documentSecurity: boolean;
             indexes: {
                 key: string;
                 type: "key" | "unique" | "fulltext";

package/dist/migrations/dataLoader.d.ts

@@ -4,7 +4,10 @@ import { z } from "zod";
 import { type Databases } from "node-appwrite";
 export declare const CollectionImportDataSchema: z.ZodObject<{
     collection: z.ZodOptional<z.ZodObject<Omit<{
+        name: z.ZodString;
         $id: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+        enabled: z.ZodDefault<z.ZodBoolean>;
+        documentSecurity: z.ZodDefault<z.ZodBoolean>;
         $createdAt: z.ZodString;
         $updatedAt: z.ZodString;
         $permissions: z.ZodDefault<z.ZodArray<z.ZodObject<{
@@ -17,10 +20,6 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             permission: string;
             target: string;
         }>, "many">>;
-        databaseId: z.ZodOptional<z.ZodString>;
-        name: z.ZodString;
-        enabled: z.ZodDefault<z.ZodBoolean>;
-        documentSecurity: z.ZodDefault<z.ZodBoolean>;
         attributes: z.ZodDefault<z.ZodArray<z.ZodUnion<[z.ZodUnion<[z.ZodUnion<[z.ZodUnion<[z.ZodUnion<[z.ZodUnion<[z.ZodUnion<[z.ZodUnion<[z.ZodUnion<[z.ZodObject<{
             key: z.ZodString;
             type: z.ZodDefault<z.ZodLiteral<"string">>;
@@ -232,14 +231,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             elements?: string[] | undefined;
             xdefault?: string | null | undefined;
         }>]>, z.ZodObject<{
-            key: z.ZodString; /**
-             * Transforms the given item based on the provided attribute mappings.
-             * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
-             *
-             * @param item - The item to be transformed.
-             * @param attributeMappings - The mappings that define how each attribute should be transformed.
-             * @returns The transformed item.
-             */
+            key: z.ZodString;
             type: z.ZodDefault<z.ZodLiteral<"relationship">>;
             error: z.ZodDefault<z.ZodString>;
             required: z.ZodDefault<z.ZodBoolean>;
@@ -483,6 +475,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 targetField: string;
             } | undefined;
         }>, "many">>;
+        databaseId: z.ZodOptional<z.ZodString>;
     }, "$createdAt" | "$updatedAt">, "strip", z.ZodTypeAny, {
         attributes: ({
             key: string;
@@ -573,12 +566,12 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
         })[];
         name: string;
         $id: string;
+        enabled: boolean;
+        documentSecurity: boolean;
         $permissions: {
             permission: string;
             target: string;
         }[];
-        enabled: boolean;
-        documentSecurity: boolean;
         indexes: {
             key: string;
             type: "key" | "unique" | "fulltext";
@@ -712,13 +705,12 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             } | undefined;
         })[] | undefined;
         $id?: string | undefined;
+        enabled?: boolean | undefined;
+        documentSecurity?: boolean | undefined;
         $permissions?: {
             permission: string;
             target: string;
         }[] | undefined;
-        databaseId?: string | undefined;
-        enabled?: boolean | undefined;
-        documentSecurity?: boolean | undefined;
         indexes?: {
             key: string;
             attributes: string[];
@@ -761,6 +753,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 targetField: string;
             } | undefined;
         }[] | undefined;
+        databaseId?: string | undefined;
     }>>;
     data: z.ZodArray<z.ZodObject<{
         rawData: z.ZodAny;
@@ -1141,12 +1134,12 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
         })[];
         name: string;
         $id: string;
+        enabled: boolean;
+        documentSecurity: boolean;
         $permissions: {
             permission: string;
             target: string;
         }[];
-        enabled: boolean;
-        documentSecurity: boolean;
         indexes: {
             key: string;
             type: "key" | "unique" | "fulltext";
@@ -1321,13 +1314,12 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
             } | undefined;
         })[] | undefined;
         $id?: string | undefined;
+        enabled?: boolean | undefined;
+        documentSecurity?: boolean | undefined;
         $permissions?: {
             permission: string;
             target: string;
         }[] | undefined;
-        databaseId?: string | undefined;
-        enabled?: boolean | undefined;
-        documentSecurity?: boolean | undefined;
         indexes?: {
             key: string;
             attributes: string[];
@@ -1370,6 +1362,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
                 targetField: string;
             } | undefined;
         }[] | undefined;
+        databaseId?: string | undefined;
     } | undefined;
 }>;
 export type CollectionImportData = z.infer<typeof CollectionImportDataSchema>;
@@ -1509,12 +1502,12 @@ export declare class DataLoader {
             })[];
             name: string;
             $id: string;
+            enabled: boolean;
+            documentSecurity: boolean;
             $permissions: {
                 permission: string;
                 target: string;
             }[];
-            enabled: boolean;
-            documentSecurity: boolean;
             indexes: {
                 key: string;
                 type: "key" | "unique" | "fulltext";

package/dist/migrations/dataLoader.js

@@ -285,28 +285,29 @@ export class DataLoader {
                                 const oldIds = Array.isArray(item.context[idMapping.sourceField])
                                     ? item.context[idMapping.sourceField]
                                     : [item.context[idMapping.sourceField]];
+                                const resolvedNewIds = [];
                                 oldIds.forEach((oldId) => {
-                                    let newIdForOldId;
-                                    // Handling users merged into a new ID
-                                    newIdForOldId = this.findNewIdForOldId(oldId, idMapping);
-                                    if (newIdForOldId) {
-                                        const targetField = idMapping.fieldToSet || idMapping.targetField;
-                                        const isArray = collectionConfig.attributes.some((attribute) => attribute.key === targetField && attribute.array);
-                                        // Properly update the target field based on whether it should be an array
-                                        if (isArray) {
-                                            if (!Array.isArray(item.finalData[targetField])) {
-                                                item.finalData[targetField] = [newIdForOldId];
-                                            }
-                                            else if (!item.finalData[targetField].includes(newIdForOldId)) {
-                                                item.finalData[targetField].push(newIdForOldId);
-                                            }
-                                        }
-                                        else {
-                                            item.finalData[targetField] = newIdForOldId;
-                                        }
-                                        needsUpdate = true;
+                                    // Attempt to find a new ID for the old ID
+                                    let newIdForOldId = this.findNewIdForOldId(oldId, idMapping);
+                                    // Check if a new ID was found and it's not already included
+                                    if (newIdForOldId &&
+                                        !resolvedNewIds.includes(newIdForOldId)) {
+                                        resolvedNewIds.push(newIdForOldId);
                                     }
                                 });
+                                if (resolvedNewIds.length) {
+                                    const targetField = idMapping.fieldToSet || idMapping.targetField;
+                                    const isArray = collectionConfig.attributes.some((attribute) => attribute.key === targetField && attribute.array);
+                                    // Set the target field based on whether it's an array or single value
+                                    if (isArray) {
+                                        item.finalData[targetField] = resolvedNewIds;
+                                    }
+                                    else {
+                                        // In case of a single value, use the first resolved ID
+                                        item.finalData[targetField] = resolvedNewIds[0];
+                                    }
+                                    needsUpdate = true;
+                                }
                             }
                         }
                     }
@@ -320,23 +321,16 @@ export class DataLoader {
         }
     }
     findNewIdForOldId(oldId, idMapping) {
-        // Check merged users first for any corresponding new ID
-        let newIdForOldId;
+        // First, check if the old ID has been merged into a new one
         for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
             if (oldIds.includes(oldId)) {
-                newIdForOldId = newUserId;
-                break;
-            }
-        }
-        // If no new ID found in merged users, check the old-to-new ID map for the target collection
-        if (!newIdForOldId) {
-            const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
-            const targetOldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(targetCollectionKey);
-            if (targetOldIdToNewIdMap && targetOldIdToNewIdMap.has(oldId)) {
-                newIdForOldId = targetOldIdToNewIdMap.get(oldId);
+                return newUserId;
             }
         }
-        return newIdForOldId;
+        // If not merged, look for a direct mapping from old to new ID
+        const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
+        const targetOldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(targetCollectionKey);
+        return targetOldIdToNewIdMap?.get(oldId);
     }
     writeMapsToJsonFile() {
         const outputDir = path.resolve(process.cwd());

package/dist/migrations/databases.d.ts

@@ -0,0 +1,2 @@
+import { Databases, type Models } from "node-appwrite";
+export declare const fetchAllDatabases: (database: Databases) => Promise<Models.Database[]>;

package/dist/migrations/databases.js

@@ -0,0 +1,23 @@
+import { Databases, Query } from "node-appwrite";
+export const fetchAllDatabases = async (database) => {
+    const databases = await database.list([Query.limit(25)]);
+    const allDatabases = databases.databases;
+    let lastDatabaseId = allDatabases[allDatabases.length - 1].$id;
+    if (databases.databases.length < 25) {
+        return allDatabases;
+    }
+    else {
+        while (lastDatabaseId) {
+            const databases = await database.list([
+                Query.limit(25),
+                Query.cursorAfter(lastDatabaseId),
+            ]);
+            allDatabases.push(...databases.databases);
+            if (databases.databases.length < 25) {
+                break;
+            }
+            lastDatabaseId = databases.databases[databases.databases.length - 1].$id;
+        }
+    }
+    return allDatabases;
+};