appwrite-utils-cli 0.0.262 → 0.0.263

package/dist/utilsController.js

@@ -11,6 +11,7 @@ import { afterImportActions, } from "./migrations/afterImportActions.js";
 import { validationRules, } from "./migrations/validationRules.js";
 import { ImportController } from "./migrations/importController.js";
 import _ from "lodash";
+import { AppwriteToX } from "./migrations/appwriteToX.js";
 async function loadConfig(configPath) {
     if (!fs.existsSync(configPath)) {
         throw new Error(`Configuration file not found at ${configPath}`);
@@ -84,6 +85,12 @@ export class UtilsController {
         if (!this.database || !this.storage || !this.config) {
             throw new Error("Database or storage not initialized");
         }
+        if (options.sync) {
+            console.log("Starting synchronization with server...");
+            const appwriteToX = new AppwriteToX(this.config, this.appwriteFolderPath);
+            await appwriteToX.toSchemas();
+            console.log("Synchronization complete, YAML and Schemas updated");
+        }
         // Start the setup
         console.log("Starting setup, this step sets up migrations, runs backup, wipes databases, and updates schemas (depending on your options)...");
         await startSetup(this.database, this.storage, this.config, options, this.appwriteFolderPath);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.262",
+  "version": "0.0.263",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/main.ts

@@ -7,6 +7,7 @@ async function main() {
   const controller = new UtilsController();
   await controller.init();
 
+  let sync = false;
   let runProd = false;
   let runStaging = false;
   let runDev = false;
@@ -17,6 +18,9 @@ async function main() {
   let importData = false;
   let wipeDocuments = false;
   let shouldWriteFile = false;
+  if (args.includes("--sync")) {
+    sync = true;
+  }
   if (args.includes("--prod")) {
     runProd = true;
   }
@@ -49,6 +53,7 @@ async function main() {
   }
   if (args.includes("--init")) {
     await controller.run({
+      sync: sync,
       runProd: runProd,
       runStaging: runStaging,
       runDev: runDev,
@@ -64,6 +69,7 @@ async function main() {
     });
   } else {
     await controller.run({
+      sync: sync,
       runProd: runProd,
       runStaging: runStaging,
       runDev: runDev,

package/src/migrations/afterImportActions.ts

@@ -13,7 +13,7 @@ import fs from "fs";
 import os from "os";
 import { logger } from "./logging.js";
 
-const getDatabaseFromConfig = (config: AppwriteConfig) => {
+export const getDatabaseFromConfig = (config: AppwriteConfig) => {
   if (!config.appwriteClient) {
     config.appwriteClient = new Client()
       .setEndpoint(config.appwriteEndpoint)
@@ -23,7 +23,7 @@ const getDatabaseFromConfig = (config: AppwriteConfig) => {
   return new Databases(config.appwriteClient!);
 };
 
-const getStorageFromConfig = (config: AppwriteConfig) => {
+export const getStorageFromConfig = (config: AppwriteConfig) => {
   if (!config.appwriteClient) {
     config.appwriteClient = new Client()
       .setEndpoint(config.appwriteEndpoint)

package/src/migrations/appwriteToX.ts

@@ -0,0 +1,122 @@
+import { SchemaGenerator } from "./schemaStrings.js";
+import { Databases, Query, type Models, type Permission } from "node-appwrite";
+import { fetchAllCollections } from "./collections.js";
+import { fetchAllDatabases } from "./databases.js";
+import {
+  collectionSchema,
+  attributeSchema,
+  type AppwriteConfig,
+  AppwriteConfigSchema,
+  type ConfigDatabases,
+  type Attribute,
+  permissionsSchema,
+  attributesSchema,
+  indexesSchema,
+} from "./schema.js";
+import { getDatabaseFromConfig } from "./afterImportActions.js";
+
+export class AppwriteToX {
+  config: AppwriteConfig;
+  updatedConfig: AppwriteConfig;
+  collToAttributeMap = new Map<string, Attribute[]>();
+  appwriteFolderPath: string;
+
+  constructor(config: AppwriteConfig, appwriteFolderPath: string) {
+    this.config = config;
+    this.updatedConfig = config;
+    this.appwriteFolderPath = appwriteFolderPath;
+  }
+
+  // Function to parse a single permission string
+  parsePermissionString = (permissionString: string) => {
+    const match = permissionString.match(/^(\w+)\('([^']+)'\)$/);
+    if (!match) {
+      throw new Error(`Invalid permission format: ${permissionString}`);
+    }
+    return {
+      permission: match[1],
+      target: match[2],
+    };
+  };
+
+  // Function to parse an array of permission strings
+  parsePermissionsArray = (permissions: string[]) => {
+    const parsedPermissions = permissionsSchema.parse(permissions);
+    // Validate the parsed permissions using Zod
+    return parsedPermissions ?? [];
+  };
+
+  updateCollectionConfigAttributes = (collection: Models.Collection) => {
+    for (const attribute of collection.attributes) {
+      const attributeMap = this.collToAttributeMap.get(
+        collection.name as string
+      );
+      const attributeParsed = attributeSchema.parse(attribute);
+      this.collToAttributeMap
+        .get(collection.name as string)
+        ?.push(attributeParsed);
+    }
+  };
+
+  async appwriteSync(config: AppwriteConfig) {
+    const db = getDatabaseFromConfig(config);
+    const databases = await fetchAllDatabases(db);
+    let updatedConfig: AppwriteConfig = { ...config };
+
+    // Loop through each database
+    for (const database of databases) {
+      const collections = await fetchAllCollections(database.$id, db);
+
+      // Loop through each collection in the current database
+      for (const collection of collections) {
+        const existingCollectionIndex = updatedConfig.collections.findIndex(
+          (c) => c.name === collection.name
+        );
+
+        // Parse the collection permissions and attributes
+        const collPermissions = this.parsePermissionsArray(
+          collection.$permissions
+        );
+        const collAttributes = attributesSchema.parse(collection.attributes);
+        this.collToAttributeMap.set(collection.name, collAttributes);
+        const collIndexes = indexesSchema.parse(collection.indexes);
+
+        // Prepare the collection object to be added or updated
+        const collToPush = collectionSchema.parse({
+          name: collection.name,
+          enabled: collection.enabled,
+          documentSecurity: collection.documentSecurity,
+          $createdAt: collection.$createdAt,
+          $updatedAt: collection.$updatedAt,
+          $permissions:
+            collPermissions.length > 0 ? collPermissions : undefined,
+          indexes: collIndexes.length > 0 ? collIndexes : undefined,
+          attributes: collAttributes.length > 0 ? collAttributes : undefined,
+        });
+
+        if (existingCollectionIndex !== -1) {
+          // Update existing collection
+          updatedConfig.collections[existingCollectionIndex] = collToPush;
+        } else {
+          // Add new collection
+          updatedConfig.collections.push(collToPush);
+        }
+      }
+
+      console.log(
+        `Processed ${collections.length} collections in ${database.name}`
+      );
+    }
+    this.updatedConfig = updatedConfig;
+  }
+
+  async toSchemas() {
+    await this.appwriteSync(this.config);
+    const generator = new SchemaGenerator(
+      this.updatedConfig,
+      this.appwriteFolderPath
+    );
+    generator.updateYamlSchemas();
+    generator.generateSchemas();
+  }
+}

package/src/migrations/dataLoader.ts

@@ -369,36 +369,37 @@ export class DataLoader {
                 )
                   ? item.context[idMapping.sourceField]
                   : [item.context[idMapping.sourceField]];
+                const resolvedNewIds: string[] = [];
 
                 oldIds.forEach((oldId: any) => {
-                  let newIdForOldId;
+                  // Attempt to find a new ID for the old ID
+                  let newIdForOldId = this.findNewIdForOldId(oldId, idMapping);
 
-                  // Handling users merged into a new ID
-                  newIdForOldId = this.findNewIdForOldId(oldId, idMapping);
-
-                  if (newIdForOldId) {
-                    const targetField =
-                      idMapping.fieldToSet || idMapping.targetField;
-                    const isArray = collectionConfig.attributes.some(
-                      (attribute) =>
-                        attribute.key === targetField && attribute.array
-                    );
-
-                    // Properly update the target field based on whether it should be an array
-                    if (isArray) {
-                      if (!Array.isArray(item.finalData[targetField])) {
-                        item.finalData[targetField] = [newIdForOldId];
-                      } else if (
-                        !item.finalData[targetField].includes(newIdForOldId)
-                      ) {
-                        item.finalData[targetField].push(newIdForOldId);
-                      }
-                    } else {
-                      item.finalData[targetField] = newIdForOldId;
-                    }
-                    needsUpdate = true;
+                  // Check if a new ID was found and it's not already included
+                  if (
+                    newIdForOldId &&
+                    !resolvedNewIds.includes(newIdForOldId)
+                  ) {
+                    resolvedNewIds.push(newIdForOldId);
                   }
                 });
+                if (resolvedNewIds.length) {
+                  const targetField =
+                    idMapping.fieldToSet || idMapping.targetField;
+                  const isArray = collectionConfig.attributes.some(
+                    (attribute) =>
+                      attribute.key === targetField && attribute.array
+                  );
+
+                  // Set the target field based on whether it's an array or single value
+                  if (isArray) {
+                    item.finalData[targetField] = resolvedNewIds;
+                  } else {
+                    // In case of a single value, use the first resolved ID
+                    item.finalData[targetField] = resolvedNewIds[0];
+                  }
+                  needsUpdate = true;
+                }
               }
             }
           }
@@ -416,29 +417,20 @@ export class DataLoader {
   }
 
   findNewIdForOldId(oldId: string, idMapping: IdMapping) {
-    // Check merged users first for any corresponding new ID
-    let newIdForOldId;
+    // First, check if the old ID has been merged into a new one
     for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
       if (oldIds.includes(oldId)) {
-        newIdForOldId = newUserId;
-        break;
-      }
-    }
-
-    // If no new ID found in merged users, check the old-to-new ID map for the target collection
-    if (!newIdForOldId) {
-      const targetCollectionKey = this.getCollectionKey(
-        idMapping.targetCollection
-      );
-      const targetOldIdToNewIdMap =
-        this.oldIdToNewIdPerCollectionMap.get(targetCollectionKey);
-
-      if (targetOldIdToNewIdMap && targetOldIdToNewIdMap.has(oldId)) {
-        newIdForOldId = targetOldIdToNewIdMap.get(oldId);
+        return newUserId;
       }
     }
 
-    return newIdForOldId;
+    // If not merged, look for a direct mapping from old to new ID
+    const targetCollectionKey = this.getCollectionKey(
+      idMapping.targetCollection
+    );
+    const targetOldIdToNewIdMap =
+      this.oldIdToNewIdPerCollectionMap.get(targetCollectionKey);
+    return targetOldIdToNewIdMap?.get(oldId);
   }
 
   private writeMapsToJsonFile() {

package/src/migrations/databases.ts

@@ -0,0 +1,25 @@
+import { Databases, Query, type Models } from "node-appwrite";
+
+export const fetchAllDatabases = async (
+  database: Databases
+): Promise<Models.Database[]> => {
+  const databases = await database.list([Query.limit(25)]);
+  const allDatabases = databases.databases;
+  let lastDatabaseId = allDatabases[allDatabases.length - 1].$id;
+  if (databases.databases.length < 25) {
+    return allDatabases;
+  } else {
+    while (lastDatabaseId) {
+      const databases = await database.list([
+        Query.limit(25),
+        Query.cursorAfter(lastDatabaseId),
+      ]);
+      allDatabases.push(...databases.databases);
+      if (databases.databases.length < 25) {
+        break;
+      }
+      lastDatabaseId = databases.databases[databases.databases.length - 1].$id;
+    }
+  }
+  return allDatabases;
+};

package/src/migrations/schema.ts

@@ -1,4 +1,4 @@
-import { ID, IndexType } from "node-appwrite";
+import { ID, IndexType, Permission } from "node-appwrite";
 import { z } from "zod";
 
 const stringAttributeSchema = z.object({
@@ -376,6 +376,8 @@ export const indexSchema = z.object({
   orders: z.array(z.string()).optional(),
 });
 
+export const indexesSchema = z.array(indexSchema);
+
 export type Index = z.infer<typeof indexSchema>;
 
 export const AttributeMappingsSchema = z.array(
@@ -510,12 +512,45 @@ export const importDefSchemas = z
   .default([])
   .describe("The import definitions for the database");
 
+export const permissionSchema = z
+  .object({
+    permission: z.string(),
+    target: z.string(),
+  })
+  .or(
+    z.string().transform((val) => {
+      const trimmedVal = val.trim();
+      // Adjusted regex to match double quotes
+      const match = trimmedVal.match(/^(\w+)\("([^"]+)"\)$/);
+      if (!match) {
+        throw new Error(`Invalid permission format: ${trimmedVal}`);
+      }
+      return {
+        permission: match[1],
+        target: match[2],
+      };
+    })
+  );
+
+export const permissionsSchema = z.array(permissionSchema).optional();
+
+export const attributesSchema = z.array(attributeSchema).default([]);
+
 export const collectionSchema = z.object({
+  name: z.string().describe("The name of the collection"),
   $id: z
     .string()
     .optional()
     .default(ID.unique())
     .describe("The ID of the collection, auto generated if not provided"),
+  enabled: z
+    .boolean()
+    .default(true)
+    .describe("Whether the collection is enabled or not"),
+  documentSecurity: z
+    .boolean()
+    .default(false)
+    .describe("Whether document security is enabled or not"),
   $createdAt: z.string(),
   $updatedAt: z.string(),
   $permissions: z
@@ -527,19 +562,6 @@ export const collectionSchema = z.object({
     )
     .default([])
     .describe("The permissions of the collection"),
-  databaseId: z
-    .string()
-    .optional()
-    .describe("The ID of the database the collection belongs to"),
-  name: z.string().describe("The name of the collection"),
-  enabled: z
-    .boolean()
-    .default(true)
-    .describe("Whether the collection is enabled or not"),
-  documentSecurity: z
-    .boolean()
-    .default(false)
-    .describe("Whether document security is enabled or not"),
   attributes: z
     .array(attributeSchema)
     .default([])
@@ -549,6 +571,10 @@ export const collectionSchema = z.object({
     .default([])
     .describe("The indexes of the collection"),
   importDefs: importDefSchemas,
+  databaseId: z
+    .string()
+    .optional()
+    .describe("The ID of the database the collection belongs to"),
 });
 
 export const CollectionCreateSchema = collectionSchema.omit({

package/src/migrations/schemaStrings.ts

@@ -7,6 +7,7 @@ import type {
 import { z } from "zod";
 import fs from "fs";
 import path from "path";
+import { dump } from "js-yaml";
 
 interface RelationshipDetail {
   parentCollection: string;
@@ -28,6 +29,16 @@ export class SchemaGenerator {
     this.extractRelationships();
   }
 
+  public updateYamlSchemas(): void {
+    // Output this.config to a YAML file at appwriteFolderPath/appwriteConfig.yaml
+    let finalConfig = this.config;
+    finalConfig.appwriteClient = null;
+    const yamlConfig = finalConfig;
+    const yamlPath = path.join(this.appwriteFolderPath, "appwriteConfig.yaml");
+    fs.writeFileSync(yamlPath, dump(yamlConfig), { encoding: "utf-8" });
+    console.log(`YAML written to ${yamlPath}`);
+  }
+
   private extractRelationships(): void {
     this.config.collections.forEach((collection) => {
       collection.attributes.forEach((attr) => {

package/src/utilsController.ts

@@ -23,6 +23,7 @@ import {
 } from "./migrations/validationRules.js";
 import { ImportController } from "./migrations/importController.js";
 import _ from "lodash";
+import { AppwriteToX } from "./migrations/appwriteToX.js";
 
 async function loadConfig(configPath: string) {
   if (!fs.existsSync(configPath)) {
@@ -35,6 +36,7 @@ async function loadConfig(configPath: string) {
 }
 
 export interface SetupOptions {
+  sync: boolean;
   runProd: boolean;
   runStaging: boolean;
   runDev: boolean;
@@ -127,6 +129,13 @@ export class UtilsController {
       throw new Error("Database or storage not initialized");
     }
 
+    if (options.sync) {
+      console.log("Starting synchronization with server...");
+      const appwriteToX = new AppwriteToX(this.config, this.appwriteFolderPath);
+      await appwriteToX.toSchemas();
+      console.log("Synchronization complete, YAML and Schemas updated");
+    }
+
     // Start the setup
     console.log(
       "Starting setup, this step sets up migrations, runs backup, wipes databases, and updates schemas (depending on your options)..."