appwrite-utils-cli 0.0.32 → 0.0.33

package/dist/migrations/dataLoader.js

@@ -234,6 +234,10 @@ export class DataLoader {
                 // Determine if this is the users collection
                 let isUsersCollection = this.getCollectionKey(this.config.usersCollectionName) ===
                     this.getCollectionKey(collection.name);
+                const collectionDefs = collection.importDefs;
+                if (!collectionDefs || !collectionDefs.length) {
+                    continue;
+                }
                 // Process create and update definitions for the collection
                 const createDefs = collection.importDefs.filter((def) => def.type === "create" || !def.type);
                 const updateDefs = collection.importDefs.filter((def) => def.type === "update");
@@ -258,8 +262,8 @@ export class DataLoader {
                 }
             }
             console.log("Running update references");
-            await this.dealWithMergedUsers();
-            await this.updateOldReferencesForNew();
+            this.dealWithMergedUsers();
+            this.updateOldReferencesForNew();
             console.log("Done running update references");
         }
         // for (const collection of this.config.collections) {
@@ -272,7 +276,7 @@ export class DataLoader {
             this.writeMapsToJsonFile();
         }
     }
-    async dealWithMergedUsers() {
+    dealWithMergedUsers() {
         const usersCollectionKey = this.getCollectionKey(this.config.usersCollectionName);
         const usersCollectionPrimaryKeyFields = new Set();
         if (!this.config.collections) {
@@ -281,7 +285,11 @@ export class DataLoader {
         // Collect primary key fields from the users collection definitions
         this.config.collections.forEach((collection) => {
             if (this.getCollectionKey(collection.name) === usersCollectionKey) {
-                collection.importDefs.forEach((importDef) => {
+                const collectionImportDefs = collection.importDefs;
+                if (!collectionImportDefs || !collectionImportDefs.length) {
+                    return;
+                }
+                collectionImportDefs.forEach((importDef) => {
                     if (importDef.primaryKeyField) {
                         usersCollectionPrimaryKeyFields.add(importDef.primaryKeyField);
                     }
@@ -293,19 +301,23 @@ export class DataLoader {
             const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
             if (!collectionData || !collectionData.data)
                 return;
-            collection.importDefs.forEach((importDef) => {
+            const collectionImportDefs = collection.importDefs;
+            if (!collectionImportDefs || !collectionImportDefs.length) {
+                return;
+            }
+            collectionImportDefs.forEach((importDef) => {
                 importDef.idMappings?.forEach((idMapping) => {
                     if (this.getCollectionKey(idMapping.targetCollection) ===
                         usersCollectionKey) {
-                        if (usersCollectionPrimaryKeyFields.has(idMapping.targetField)) {
+                        const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
+                        if (usersCollectionPrimaryKeyFields.has(targetFieldKey)) {
                             // Process each item in the collection
                             collectionData.data.forEach((item) => {
                                 const oldId = item.context[idMapping.sourceField];
                                 const newId = this.mergedUserMap.get(oldId);
                                 if (newId) {
                                     // Update context to use new user ID
-                                    item.context[idMapping.fieldToSet || idMapping.targetField] =
-                                        newId;
+                                    item.context[idMapping.fieldToSet || targetFieldKey] = newId;
                                 }
                             });
                         }
@@ -333,123 +345,82 @@ export class DataLoader {
                             for (const idMapping of importDef.idMappings) {
                                 const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
                                 const fieldToSetKey = idMapping.fieldToSet || idMapping.sourceField;
+                                const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
                                 const valueToMatch = collectionData.data[i].context[idMapping.sourceField];
+                                // Skip if value to match is missing or empty
                                 if (!valueToMatch || _.isEmpty(valueToMatch))
                                     continue;
+                                const isFieldToSetArray = collectionConfig.attributes.find((attribute) => attribute.key === fieldToSetKey)?.array;
                                 const targetCollectionData = this.importMap.get(targetCollectionKey);
                                 if (!targetCollectionData || !targetCollectionData.data)
                                     continue;
-                                const foundData = targetCollectionData.data.filter((data) => {
-                                    const targetValue = data.context[idMapping.targetField];
+                                // Find matching data in the target collection
+                                const foundData = targetCollectionData.data.filter(({ context }) => {
+                                    const targetValue = context[targetFieldKey];
                                     const isMatch = `${targetValue}` === `${valueToMatch}`;
-                                    // Debugging output to understand what's being compared
-                                    logger.warn(`Comparing target: ${targetValue} with match: ${valueToMatch} - Result: ${isMatch}`);
-                                    return isMatch;
+                                    // Ensure the targetValue is defined and not null
+                                    return (isMatch &&
+                                        targetValue !== undefined &&
+                                        targetValue !== null);
                                 });
+                                // Log and skip if no matching data found
                                 if (!foundData.length) {
-                                    console.log(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey}`);
+                                    console.log(`No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`);
                                     logger.error(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
                                     continue;
                                 }
                                 needsUpdate = true;
-                                // Properly handle arrays and non-arrays
-                                if (Array.isArray(collectionData.data[i].finalData[fieldToSetKey])) {
-                                    collectionData.data[i].finalData[fieldToSetKey] =
-                                        foundData.map((data) => data.finalData);
+                                const getCurrentDataFiltered = (currentData) => {
+                                    if (Array.isArray(currentData.finalData[fieldToSetKey])) {
+                                        return currentData.finalData[fieldToSetKey].filter((data) => `${data}` !== `${valueToMatch}`);
+                                    }
+                                    return currentData.finalData[fieldToSetKey];
+                                };
+                                // Get the current data to be updated
+                                const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
+                                // Extract the new data to set
+                                const newData = foundData.map((data) => data.context[idMapping.targetField]);
+                                // Handle cases where current data is an array
+                                if (isFieldToSetArray) {
+                                    if (!currentDataFiltered) {
+                                        // Set new data if current data is undefined
+                                        collectionData.data[i].finalData[fieldToSetKey] =
+                                            Array.isArray(newData) ? newData : [newData];
+                                    }
+                                    else {
+                                        // Merge arrays if new data is non-empty array and filter for uniqueness
+                                        collectionData.data[i].finalData[fieldToSetKey] = [
+                                            ...new Set([...currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}`)),
+                                        ];
+                                    }
                                 }
                                 else {
-                                    collectionData.data[i].finalData[fieldToSetKey] =
-                                        foundData[0].finalData;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            if (needsUpdate) {
-                this.importMap.set(collectionKey, collectionData);
-            }
-        }
-    }
-    async updateReferencesInRelatedCollections() {
-        if (!this.config.collections) {
-            return;
-        }
-        // Iterate over each collection configuration
-        for (const collectionConfig of this.config.collections) {
-            const collectionKey = this.getCollectionKey(collectionConfig.name);
-            const collectionData = this.importMap.get(collectionKey);
-            if (!collectionData || !collectionData.data)
-                continue;
-            console.log(`Updating references for collection: ${collectionConfig.name}`);
-            // Iterate over each data item in the current collection
-            for (const item of collectionData.data) {
-                let needsUpdate = false;
-                // Check if the current collection has import definitions with idMappings
-                if (collectionConfig.importDefs) {
-                    for (const importDef of collectionConfig.importDefs) {
-                        if (importDef.idMappings) {
-                            // Iterate over each idMapping defined for the current import definition
-                            for (const idMapping of importDef.idMappings) {
-                                const oldIds = Array.isArray(item.context[idMapping.sourceField])
-                                    ? item.context[idMapping.sourceField]
-                                    : [item.context[idMapping.sourceField]];
-                                const resolvedNewIds = [];
-                                oldIds.forEach((oldId) => {
-                                    // Attempt to find a new ID for the old ID
-                                    let newIdForOldId = this.findNewIdForOldId(oldId, idMapping, importDef);
-                                    if (newIdForOldId &&
-                                        !resolvedNewIds.includes(newIdForOldId)) {
-                                        resolvedNewIds.push(newIdForOldId);
+                                    if (!currentDataFiltered) {
+                                        // Set new data if current data is undefined
+                                        collectionData.data[i].finalData[fieldToSetKey] =
+                                            Array.isArray(newData) ? newData : [newData];
                                     }
-                                    else {
-                                        logger.error(`No new ID found for old ID ${oldId} in collection ${collectionConfig.name}`);
+                                    else if (Array.isArray(newData) && newData.length > 0) {
+                                        // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
+                                        collectionData.data[i].finalData[fieldToSetKey] = [
+                                            ...new Set([currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}`)),
+                                        ];
+                                    }
+                                    else if (!Array.isArray(newData) && newData !== undefined) {
+                                        // Simply update the field if new data is not an array and defined
+                                        collectionData.data[i].finalData[fieldToSetKey] = newData;
                                     }
-                                });
-                                if (resolvedNewIds.length) {
-                                    const targetField = idMapping.fieldToSet || idMapping.targetField;
-                                    const isArray = collectionConfig.attributes.some((attribute) => attribute.key === targetField && attribute.array);
-                                    // Set the target field based on whether it's an array or single value
-                                    item.finalData[targetField] = isArray
-                                        ? resolvedNewIds
-                                        : resolvedNewIds[0];
-                                    needsUpdate = true;
                                 }
                             }
                         }
                     }
                 }
-                // Update the importMap if changes were made to the item
-                if (needsUpdate) {
-                    this.importMap.set(collectionKey, collectionData);
-                    logger.info(`Updated item: ${JSON.stringify(item.finalData, undefined, 2)}`);
-                }
             }
-        }
-    }
-    findNewIdForOldId(oldId, idMapping, importDef) {
-        // First, check if this ID mapping is related to the users collection.
-        const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
-        const isUsersCollection = targetCollectionKey ===
-            this.getCollectionKey(this.config.usersCollectionName);
-        // If handling users, check the mergedUserMap for any existing new ID.
-        if (isUsersCollection) {
-            for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
-                if (oldIds.includes(oldId)) {
-                    return newUserId;
-                }
-            }
-        }
-        // If not a user or no merged ID found, check the regular ID mapping from old to new.
-        const targetCollectionData = this.importMap.get(targetCollectionKey);
-        if (targetCollectionData) {
-            const foundEntry = targetCollectionData.data.find((entry) => entry.context[importDef.primaryKeyField] === oldId);
-            if (foundEntry) {
-                return foundEntry.context.docId; // Assuming `docId` stores the new ID after import
+            // Update the import map if any changes were made
+            if (needsUpdate) {
+                this.importMap.set(collectionKey, collectionData);
             }
         }
-        logger.error(`No corresponding new ID found for ${oldId} in ${targetCollectionKey}`);
-        return null; // Return null if no new ID is found
     }
     writeMapsToJsonFile() {
         const outputDir = path.resolve(process.cwd());
@@ -601,12 +572,12 @@ export class DataLoader {
             if (!existingId) {
                 // No existing user ID, generate a new unique ID
                 existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
-                transformedItem.userId = existingId; // Assign the new ID to the transformed data's userId field
+                transformedItem.docId = existingId; // Assign the new ID to the transformed data's docId field
             }
             // Create a context object for the item, including the new ID
             let context = this.createContext(db, collection, item, existingId);
             // Merge the transformed data into the context
-            context = { ...context, ...transformedItem };
+            context = { ...context, ...transformedItem, ...userData.finalData };
             // If a primary key field is defined, handle the ID mapping and check for duplicates
             if (importDef.primaryKeyField) {
                 const oldId = item[importDef.primaryKeyField];

package/dist/migrations/schemaStrings.js

@@ -17,25 +17,25 @@ export class SchemaGenerator {
         const collections = this.config.collections;
         delete this.config.collections;
         const configPath = path.join(this.appwriteFolderPath, "appwriteConfig.ts");
-        const configContent = `import { type AppwriteConfig } from "appwrite-utils";
-  
-  const appwriteConfig: AppwriteConfig = {
-    appwriteEndpoint: "${this.config.appwriteEndpoint}",
-    appwriteProject: "${this.config.appwriteProject}",
-    appwriteKey: "${this.config.appwriteKey}",
-    enableDevDatabase: ${this.config.enableDevDatabase},
-    enableBackups: ${this.config.enableBackups},
-    backupInterval: ${this.config.backupInterval},
-    backupRetention: ${this.config.backupRetention},
-    enableBackupCleanup: ${this.config.enableBackupCleanup},
-    enableMockData: ${this.config.enableMockData},
-    enableWipeOtherDatabases: ${this.config.enableWipeOtherDatabases},
-    documentBucketId: "${this.config.documentBucketId}",
-    usersCollectionName: "${this.config.usersCollectionName}",
-    databases: ${JSON.stringify(this.config.databases)}
-  };
-  
-  export default appwriteConfig;
+        const configContent = `import { type AppwriteConfig } from "appwrite-utils";
+  
+  const appwriteConfig: AppwriteConfig = {
+    appwriteEndpoint: "${this.config.appwriteEndpoint}",
+    appwriteProject: "${this.config.appwriteProject}",
+    appwriteKey: "${this.config.appwriteKey}",
+    enableDevDatabase: ${this.config.enableDevDatabase},
+    enableBackups: ${this.config.enableBackups},
+    backupInterval: ${this.config.backupInterval},
+    backupRetention: ${this.config.backupRetention},
+    enableBackupCleanup: ${this.config.enableBackupCleanup},
+    enableMockData: ${this.config.enableMockData},
+    enableWipeOtherDatabases: ${this.config.enableWipeOtherDatabases},
+    documentBucketId: "${this.config.documentBucketId}",
+    usersCollectionName: "${this.config.usersCollectionName}",
+    databases: ${JSON.stringify(this.config.databases)}
+  };
+  
+  export default appwriteConfig;
   `;
         fs.writeFileSync(configPath, configContent, { encoding: "utf-8" });
         const collectionsFolderPath = path.join(this.appwriteFolderPath, "collections");
@@ -45,19 +45,19 @@ export class SchemaGenerator {
         collections?.forEach((collection) => {
             const { databaseId, ...collectionWithoutDbId } = collection; // Destructure to exclude databaseId
             const collectionFilePath = path.join(collectionsFolderPath, `${collection.name}.ts`);
-            const collectionContent = `import { type CollectionCreate } from "appwrite-utils";
-  
-  const ${collection.name}Config: Partial<CollectionCreate> = {
-    name: "${collection.name}",
-    $id: "${collection.$id}",
-    enabled: ${collection.enabled},
-    documentSecurity: ${collection.documentSecurity},
-    $permissions: [
+            const collectionContent = `import { type CollectionCreate } from "appwrite-utils";
+  
+  const ${collection.name}Config: Partial<CollectionCreate> = {
+    name: "${collection.name}",
+    $id: "${collection.$id}",
+    enabled: ${collection.enabled},
+    documentSecurity: ${collection.documentSecurity},
+    $permissions: [
       ${collection.$permissions
                 .map((permission) => `{ permission: "${permission.permission}", target: "${permission.target}" }`)
-                .join(",\n    ")}
-    ],
-    attributes: [
+                .join(",\n    ")}
+    ],
+    attributes: [
       ${collection.attributes
                 .map((attr) => {
                 return `{ ${Object.entries(attr)
@@ -85,9 +85,9 @@ export class SchemaGenerator {
                 })
                     .join(", ")} }`;
             })
-                .join(",\n    ")}
-    ],
-    indexes: [
+                .join(",\n    ")}
+    ],
+    indexes: [
       ${(collection.indexes?.map((index) => {
                 // Map each attribute to ensure it is properly quoted
                 const formattedAttributes = index.attributes.map((attr) => `"${attr}"`).join(", ") ?? "";
@@ -95,11 +95,11 @@ export class SchemaGenerator {
                     ?.filter((order) => order !== null)
                     .map((order) => `"${order}"`)
                     .join(", ") ?? ""}] }`;
-            }) ?? []).join(",\n    ")}
-    ]
-  };
-  
-  export default ${collection.name}Config;
+            }) ?? []).join(",\n    ")}
+    ]
+  };
+  
+  export default ${collection.name}Config;
   `;
             fs.writeFileSync(collectionFilePath, collectionContent, {
                 encoding: "utf-8",

package/dist/migrations/users.js

@@ -69,6 +69,9 @@ export class UsersController {
     async getAllUsers() {
         const allUsers = [];
         const users = await this.users.list([Query.limit(200)]);
+        if (users.users.length === 0) {
+            return [];
+        }
         if (users.users.length === 200) {
             let lastDocumentId = users.users[users.users.length - 1].$id;
             allUsers.push(...users.users);

package/package.json

@@ -1,54 +1,54 @@
-{
-  "name": "appwrite-utils-cli",
-  "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.32",
-  "main": "src/main.ts",
-  "type": "module",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/zachhandley/AppwriteUtils"
-  },
-  "author": "Zach Handley <zach@blackleafdigital.com> (https://zachhandley.com)",
-  "keywords": [
-    "appwrite",
-    "cli",
-    "utils",
-    "migrations",
-    "data",
-    "database",
-    "import",
-    "migration",
-    "utility"
-  ],
-  "bin": {
-    "appwrite-init": "./dist/init.js",
-    "appwrite-migrate": "./dist/main.js"
-  },
-  "scripts": {
-    "build": "bun run tsc",
-    "init": "tsx --no-cache src/init.ts",
-    "migrate": "tsx --no-cache src/main.ts",
-    "deploy": "bun run build && npm publish --access public",
-    "postinstall": "echo 'This package is intended for CLI use only and should not be added as a dependency in other projects.'"
-  },
-  "dependencies": {
-    "@types/inquirer": "^9.0.7",
-    "appwrite-utils": "^0.2.4",
-    "commander": "^12.0.0",
-    "inquirer": "^9.2.20",
-    "js-yaml": "^4.1.0",
-    "lodash": "^4.17.21",
-    "luxon": "^3.4.4",
-    "nanostores": "^0.10.3",
-    "node-appwrite": "^12.0.1",
-    "tsx": "^4.9.3",
-    "winston": "^3.13.0",
-    "zod": "^3.22.4"
-  },
-  "devDependencies": {
-    "@types/js-yaml": "^4.0.9",
-    "@types/lodash": "^4.17.0",
-    "@types/luxon": "^3.4.2",
-    "typescript": "^5.0.0"
-  }
-}
+{
+  "name": "appwrite-utils-cli",
+  "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
+  "version": "0.0.33",
+  "main": "src/main.ts",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/zachhandley/AppwriteUtils"
+  },
+  "author": "Zach Handley <zach@blackleafdigital.com> (https://zachhandley.com)",
+  "keywords": [
+    "appwrite",
+    "cli",
+    "utils",
+    "migrations",
+    "data",
+    "database",
+    "import",
+    "migration",
+    "utility"
+  ],
+  "bin": {
+    "appwrite-init": "./dist/init.js",
+    "appwrite-migrate": "node --max-old-space-size=16384 ./dist/main.js"
+  },
+  "scripts": {
+    "build": "bun run tsc",
+    "init": "tsx --no-cache src/init.ts",
+    "migrate": "tsx --no-cache src/main.ts",
+    "deploy": "bun run build && npm publish --access public",
+    "postinstall": "echo 'This package is intended for CLI use only and should not be added as a dependency in other projects.'"
+  },
+  "dependencies": {
+    "@types/inquirer": "^9.0.7",
+    "appwrite-utils": "^0.2.5",
+    "commander": "^12.0.0",
+    "inquirer": "^9.2.20",
+    "js-yaml": "^4.1.0",
+    "lodash": "^4.17.21",
+    "luxon": "^3.4.4",
+    "nanostores": "^0.10.3",
+    "node-appwrite": "^12.0.1",
+    "tsx": "^4.9.3",
+    "winston": "^3.13.0",
+    "zod": "^3.22.4"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/lodash": "^4.17.0",
+    "@types/luxon": "^3.4.2",
+    "typescript": "^5.0.0"
+  }
+}