@memberjunction/codegen-lib 2.29.2 → 2.30.0

package/dist/Database/manage-metadata.js

@@ -33,7 +33,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 var ManageMetadataBase_1;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ManageMetadataBase = exports.FieldValidatorResult = void 0;
+exports.ManageMetadataBase = exports.ValidatorResult = void 0;
 const config_1 = require("../Config/config");
 const core_1 = require("@memberjunction/core");
 const status_logging_1 = require("../Misc/status_logging");
@@ -44,19 +44,27 @@ const uuid_1 = require("uuid");
 const fs = __importStar(require("fs"));
 const path_1 = __importDefault(require("path"));
 const sql_logging_1 = require("../Misc/sql_logging");
-class FieldValidatorResult {
+const aiengine_1 = require("@memberjunction/aiengine");
+class ValidatorResult {
     constructor() {
         this.entityName = "";
-        this.fieldName = "";
         this.sourceCheckConstraint = "";
         this.functionText = "";
         this.functionName = "";
         this.functionDescription = "";
+        /**
+         * The ID value in the Generated Codes entity that was created for this validator.
+         */
+        this.generatedCodeId = "";
+        /**
+         * The ID for the AI Model that was used to generate the code
+         */
+        this.aiModelID = "";
         this.wasGenerated = true;
         this.success = false;
     }
 }
-exports.FieldValidatorResult = FieldValidatorResult;
+exports.ValidatorResult = ValidatorResult;
 /**
  * Base class for managing metadata within the CodeGen system. This class can be sub-classed to extend/override base class functionality. Make sure to use the RegisterClass decorator from the @memberjunction/global package
  * to properly register your subclass with a priority of 1+ to ensure it gets instantiated.
@@ -81,17 +89,17 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
         return this._modifiedEntityList;
     }
     /**
-     * Globally scoped list of field validators that have been generated during the metadata management process.
+     * Globally scoped list of validators that have been generated during the metadata management process.
      */
-    static get generatedFieldValidators() {
-        return this._generatedFieldValidators;
+    static get generatedValidators() {
+        return this._generatedValidators;
     }
     /**
      * Primary function to manage metadata within the CodeGen system. This function will call a series of sub-functions to manage the metadata.
      * @param ds - the DataSource object to use for querying and updating the database
      * @returns
      */
-    async manageMetadata(ds) {
+    async manageMetadata(ds, currentUser) {
         const md = new core_1.Metadata();
         const excludeSchemas = config_1.configInfo.excludeSchemas ? config_1.configInfo.excludeSchemas : [];
         let bSuccess = true;
@@ -131,7 +139,7 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
         (0, status_logging_1.logStatus)(`    > Recompiled base views in ${(new Date().getTime() - start.getTime()) / 1000} seconds`);
         start = new Date();
         (0, status_logging_1.logStatus)('   Managing entity fields...');
-        if (!await this.manageEntityFields(ds, excludeSchemas, false, false)) {
+        if (!await this.manageEntityFields(ds, excludeSchemas, false, false, currentUser)) {
             (0, status_logging_1.logError)('   Error managing entity fields');
             bSuccess = false;
         }
@@ -472,7 +480,7 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
      * @param excludeSchemas
      * @returns
      */
-    async manageEntityFields(ds, excludeSchemas, skipCreatedAtUpdatedAtDeletedAtFieldValidation, skipEntityFieldValues) {
+    async manageEntityFields(ds, excludeSchemas, skipCreatedAtUpdatedAtDeletedAtFieldValidation, skipEntityFieldValues, currentUser) {
         let bSuccess = true;
         const startTime = new Date();
         if (!skipCreatedAtUpdatedAtDeletedAtFieldValidation) {
@@ -516,7 +524,7 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
         if (!skipEntityFieldValues) {
             const step6StartTime = new Date();
             (0, status_logging_1.logStatus)(`      Starting to manage entity field values...`);
-            if (!await this.manageEntityFieldValues(ds, excludeSchemas)) {
+            if (!await this.manageEntityFieldValuesAndValidatorFunctions(ds, excludeSchemas, currentUser, false)) {
                 (0, status_logging_1.logError)('Error managing entity field values');
                 bSuccess = false;
             }
@@ -1158,7 +1166,7 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
             return false;
         }
     }
-    async manageEntityFieldValues(ds, excludeSchemas) {
+    async manageEntityFieldValuesAndValidatorFunctions(ds, excludeSchemas, currentUser, skipDBUpdate) {
         try {
             // here we want to get all of the entity fields that have check constraints attached to them. For each field that has a check constraint, we want to
             // evaluate it to see if it is a simple series of OR statements or not, if it is a simple series of OR statements, we can parse the possible values
@@ -1169,36 +1177,57 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
             const result = await ds.query(sSQL);
             const efvSQL = `SELECT * FROM [${(0, config_1.mj_core_schema)()}].EntityFieldValue`;
             const allEntityFieldValues = await ds.query(efvSQL);
+            const efSQL = `SELECT * FROM [${(0, config_1.mj_core_schema)()}].vwEntityFields ORDER BY EntityID, Sequence`;
+            const allEntityFields = await ds.query(efSQL);
             const generationPromises = [];
-            // now, for each of the constraints we get back here, loop through and evaluate if they're simple and if they're simple, parse and sync with entity field values for that field
-            for (const r of result) {
+            const columnLevelResults = result.filter((r) => r.EntityFieldID); // get the column level constraints
+            const tableLevelResults = result.filter((r) => !r.EntityFieldID); // get the table level constraints
+            for (const r of columnLevelResults) {
+                // now, for each of the constraints we get back here, loop through and evaluate if they're simple and if they're simple, parse and sync with entity field values for that field
                 if (r.ConstraintDefinition && r.ConstraintDefinition.length > 0) {
                     const parsedValues = this.parseCheckConstraintValues(r.ConstraintDefinition, r.ColumnName, r.EntityName);
                     if (parsedValues) {
-                        // flip the order of parsedValues because they come out in reverse order from SQL Server
-                        parsedValues.reverse();
-                        // we have parsed values from the check constraint, so sync them with the entity field values
-                        await this.syncEntityFieldValues(ds, r.EntityFieldID, parsedValues, allEntityFieldValues);
-                        // finally, make sure the ValueListType column within the EntityField table is set to "List" because for check constraints we only allow the values specified in the list.
-                        // check to see if the ValueListType is already set to "List", if not, update it
-                        const sSQLCheck = `SELECT ValueListType FROM [${(0, config_1.mj_core_schema)()}].EntityField WHERE ID='${r.EntityFieldID}'`;
-                        const checkResult = await ds.query(sSQLCheck);
-                        if (checkResult && checkResult.length > 0 && checkResult[0].ValueListType.trim().toLowerCase() !== 'list') {
-                            const sSQL = `UPDATE [${(0, config_1.mj_core_schema)()}].EntityField SET ValueListType='List' WHERE ID='${r.EntityFieldID}'`;
-                            await this.LogSQLAndExecute(ds, sSQL, `SQL text to update ValueListType for entity field ID ${r.EntityFieldID}`);
+                        if (!skipDBUpdate) {
+                            // we only do this part if we are not skiping the database update as this code will sync values from the CHECK
+                            // with the EntityFieldValues in the database.
+                            // 1st, flip the order of parsedValues because they come out in reverse order from SQL Server
+                            parsedValues.reverse();
+                            // we have parsed values from the check constraint, so sync them with the entity field values
+                            await this.syncEntityFieldValues(ds, r.EntityFieldID, parsedValues, allEntityFieldValues);
+                            // finally, make sure the ValueListType column within the EntityField table is set to "List" because for check constraints we only allow the values specified in the list.
+                            // check to see if the ValueListType is already set to "List", if not, update it
+                            const sSQLCheck = `SELECT ValueListType FROM [${(0, config_1.mj_core_schema)()}].EntityField WHERE ID='${r.EntityFieldID}'`;
+                            const checkResult = await ds.query(sSQLCheck);
+                            if (checkResult && checkResult.length > 0 && checkResult[0].ValueListType.trim().toLowerCase() !== 'list') {
+                                const sSQL = `UPDATE [${(0, config_1.mj_core_schema)()}].EntityField SET ValueListType='List' WHERE ID='${r.EntityFieldID}'`;
+                                await this.LogSQLAndExecute(ds, sSQL, `SQL text to update ValueListType for entity field ID ${r.EntityFieldID}`);
+                            }
+                        }
+                        else {
+                            // we are skipping the DB update, nothing to do, eh?
                         }
                     }
                     else {
                         // if we get here that means we don't have a simple condition in the check constraint that the RegEx could parse. If Advanced Generation is enabled, we will
                         // attempt to use an LLM to do things fancier now
-                        if (config_1.configInfo.advancedGeneration?.enableAdvancedGeneration && config_1.configInfo.advancedGeneration?.features.find(f => f.name === 'ParseCheckConstraints' && f.enabled)) {
+                        if (config_1.configInfo.advancedGeneration?.enableAdvancedGeneration &&
+                            config_1.configInfo.advancedGeneration?.features.find(f => f.name === 'ParseCheckConstraints' && f.enabled)) {
                             // the user has the feature turned on, let's generate a description of the constraint and then build a Validate function for the constraint 
                             // run this in parallel
-                            generationPromises.push(this.runValidationGeneration(r));
+                            generationPromises.push(this.runValidationGeneration(r, allEntityFields, !skipDBUpdate, currentUser));
                         }
                     }
                 }
             }
+            // now for the table level constraints run the process for advanced generation
+            for (const r of tableLevelResults) {
+                if (config_1.configInfo.advancedGeneration?.enableAdvancedGeneration &&
+                    config_1.configInfo.advancedGeneration?.features.find(f => f.name === 'ParseCheckConstraints' && f.enabled)) {
+                    // the user has the feature turned on, let's generate a description of the constraint and then build a Validate function for the constraint 
+                    // run this in parallel
+                    generationPromises.push(this.runValidationGeneration(r, allEntityFields, !skipDBUpdate, currentUser));
+                }
+            }
             // await the completion of all generation promises here
             await Promise.all(generationPromises);
             return true;
@@ -1208,21 +1237,37 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
             return false;
         }
     }
-    async runValidationGeneration(r) {
-        const generatedFunction = await this.generateFieldValidatorFunctionFromCheckConstraint(r);
+    /**
+     * This method will load all generated code from the database - this is intended to be used when you are bypassing managing the metadata.
+     * @param ds
+     * @param currentUser
+     */
+    async loadGeneratedCode(ds, currentUser) {
+        try {
+            // right now we're just doing validator functions which are handled here
+            return await this.manageEntityFieldValuesAndValidatorFunctions(ds, [], currentUser, true);
+        }
+        catch (e) {
+            (0, status_logging_1.logError)(e);
+            return false;
+        }
+    }
+    async runValidationGeneration(r, allEntityFields, generateNewCode, currentUser) {
+        const generatedFunction = await this.generateValidatorFunctionFromCheckConstraint(r, allEntityFields, currentUser, generateNewCode);
         if (generatedFunction?.success) {
             // LLM was able to generate a function for us, so let's store it in the static array, will be used later when we emit the BaseEntity sub-class
-            ManageMetadataBase_1._generatedFieldValidators.push(generatedFunction);
+            ManageMetadataBase_1._generatedValidators.push(generatedFunction);
         }
     }
     /**
      * Generates a TypeScript field validator function from the text of a SQL CHECK constraint.
-     * @param entityName
-     * @param fieldName
-     * @param constraintDefinition
+     * @param data - the data object containing the entity name, column name, and constraint definition
+     * @param allEntityFields - all of the entity fields in the system
+     * @param currentUser - the current user
+     * @param generateNewCode - a flag indicating whether or not to generate new code, this is set to false when we are just loading the generated code from the database.
      * @returns a data structure with the function text, function name, function description, and a success flag
      */
-    async generateFieldValidatorFunctionFromCheckConstraint(data) {
+    async generateValidatorFunctionFromCheckConstraint(data, allEntityFields, currentUser, generateNewCode) {
         const entityName = data.EntityName;
         const fieldName = data.ColumnName;
         const constraintDefinition = data.ConstraintDefinition;
@@ -1230,10 +1275,11 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
         const generatedValidationFunctionDescription = data.GeneratedValidationFunctionDescription;
         const generatedValidationFunctionCode = data.GeneratedValidationFunctionCode;
         const generatedValidationFunctionCheckConstraint = data.GeneratedValidationFunctionCheckConstraint;
-        const returnResult = new FieldValidatorResult();
+        const returnResult = new ValidatorResult();
         returnResult.success = false;
         returnResult.entityName = entityName;
         returnResult.fieldName = fieldName;
+        returnResult.generatedCodeId = data.GeneratedCodeID; // this came from the database, so we'll store it here for reference so we update the record later instead of creating a new one
         returnResult.sourceCheckConstraint = constraintDefinition;
         if (generatedValidationFunctionCheckConstraint === constraintDefinition) {
             // in this situation, we have an EXACT match of the previous version of a CHECK constraint and what is now the CHECK constraint - meaning it hasn't changed
@@ -1248,21 +1294,32 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
             }
         }
         try {
-            if (config_1.configInfo.advancedGeneration?.enableAdvancedGeneration && config_1.configInfo.advancedGeneration?.features.find(f => f.name === 'ParseCheckConstraints' && f.enabled)) {
+            if (generateNewCode && config_1.configInfo.advancedGeneration?.enableAdvancedGeneration && config_1.configInfo.advancedGeneration?.features.find(f => f.name === 'ParseCheckConstraints' && f.enabled)) {
                 // feature is enabled, so let's call the LLM to generate a function for us
                 const ag = new advanced_generation_1.AdvancedGeneration();
                 const llm = ag.LLM;
                 if (!llm) {
                     // we weren't able to get an LLM instance, so log an error and return
-                    (0, status_logging_1.logError)('   >>> Error generating field validator function from check constraint. Unable to get an LLM instance.');
+                    (0, status_logging_1.logError)('   >>> Error generating validator function from check constraint. Unable to get an LLM instance.');
                     return returnResult;
                 }
+                await aiengine_1.AIEngine.Instance.Config(false, currentUser); // make sure metadata loaded
+                const model = aiengine_1.AIEngine.Instance.Models.find(m => m.APINameOrName.trim().toLowerCase() === ag.AIModel.trim().toLowerCase() &&
+                    m.Vendor?.trim().toLowerCase() === ag.AIVendor.trim().toLowerCase());
+                if (!model)
+                    throw new Error(`   >>> Error generating validator function from check constraint. Unable to find AI Model with name ${ag.AIModel} and vendor ${ag.AIVendor}.`);
                 const prompt = ag.getPrompt('CheckConstraintParser');
+                const entityFieldListInfo = allEntityFields.filter(item => item.Entity.trim().toLowerCase() === data.EntityName.trim().toLowerCase()).map(item => `   * ${item.Name} - ${item.Type}`).join('\n');
+                const existingMethodNameBlock = generatedValidationFunctionName ? `Existing Method Name: ${generatedValidationFunctionName}\n Please reuse this SAME method name for the new generation` : '';
+                const markedUpSysPrompt = ag.fillTemplate(prompt.systemPrompt, {
+                    ENTITY_FIELD_LIST: entityFieldListInfo,
+                    EXISTING_METHOD_NAME: existingMethodNameBlock
+                }); // prompt.systemPrompt.replace(/{{ENTITY_FIELD_LIST}}/g, entityFieldListInfo);
                 const result = await llm.ChatCompletion({
                     messages: [
                         {
                             role: 'system',
-                            content: prompt.systemPrompt
+                            content: markedUpSysPrompt
                         },
                         {
                             role: 'user',
@@ -1279,6 +1336,7 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
                         returnResult.functionText = structuredResult.Code;
                         returnResult.functionName = structuredResult.MethodName;
                         returnResult.functionDescription = structuredResult.Description;
+                        returnResult.aiModelID = model.ID;
                         returnResult.wasGenerated = true; // we just generated this code
                         returnResult.success = true;
                     }
@@ -1450,53 +1508,80 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
     async createNewEntityName(newEntity) {
         const ag = new advanced_generation_1.AdvancedGeneration();
         if (ag.featureEnabled('EntityNames')) {
-            // get the LLM for this entity
-            const chat = ag.LLM;
-            const prompt = ag.getPrompt('EntityNames');
-            const systemPrompt = ag.fillTemplate(prompt.systemPrompt, newEntity);
-            const userMessage = ag.fillTemplate(prompt.userMessage, newEntity);
-            const result = await chat.ChatCompletion({
-                model: ag.AIModel,
-                messages: [
-                    {
-                        role: 'system',
-                        content: systemPrompt
-                    },
-                    {
-                        role: 'user',
-                        content: userMessage
-                    }
-                ]
-            });
-            if (result?.success) {
-                const resultText = result?.data.choices[0].message.content;
-                try {
-                    const structuredResult = JSON.parse(resultText);
-                    if (structuredResult?.entityName) {
-                        return structuredResult.entityName;
-                    }
-                    else {
-                        console.warn('   >>> Advanced Generation Error: LLM returned a blank entity name, falling back to simple generated entity name');
-                        return this.simpleNewEntityName(newEntity.TableName);
-                    }
+            return this.newEntityNameWithAdvancedGeneration(ag, newEntity);
+        }
+        else {
+            return this.simpleNewEntityName(newEntity.SchemaName, newEntity.TableName);
+        }
+    }
+    async newEntityNameWithAdvancedGeneration(ag, newEntity) {
+        // get the LLM for this entity
+        const chat = ag.LLM;
+        const prompt = ag.getPrompt('EntityNames');
+        const systemPrompt = ag.fillTemplate(prompt.systemPrompt, newEntity);
+        const userMessage = ag.fillTemplate(prompt.userMessage, newEntity);
+        const result = await chat.ChatCompletion({
+            model: ag.AIModel,
+            messages: [
+                {
+                    role: 'system',
+                    content: systemPrompt
+                },
+                {
+                    role: 'user',
+                    content: userMessage
                 }
-                catch (e) {
-                    console.warn('   >>> Advanced Generation Error: LLM returned invalid result, falling back to simple generated entity name. Result from LLM: ' + resultText, e);
-                    return this.simpleNewEntityName(newEntity.TableName);
+            ]
+        });
+        if (result?.success) {
+            const resultText = result?.data.choices[0].message.content;
+            try {
+                const structuredResult = JSON.parse(resultText);
+                if (structuredResult?.entityName) {
+                    return this.markupEntityName(newEntity.SchemaName, structuredResult.entityName);
+                }
+                else {
+                    console.warn('   >>> Advanced Generation Error: LLM returned a blank entity name, falling back to simple generated entity name');
+                    return this.simpleNewEntityName(newEntity.SchemaName, newEntity.TableName);
                 }
             }
-            else {
-                console.warn('   >>> Advanced Generation Error: LLM call failed, falling back to simple generated entity name.');
-                return this.simpleNewEntityName(newEntity.TableName);
+            catch (e) {
+                console.warn('   >>> Advanced Generation Error: LLM returned invalid result, falling back to simple generated entity name. Result from LLM: ' + resultText, e);
+                return this.simpleNewEntityName(newEntity.SchemaName, newEntity.TableName);
             }
         }
         else {
-            return this.simpleNewEntityName(newEntity.TableName);
+            console.warn('   >>> Advanced Generation Error: LLM call failed, falling back to simple generated entity name.');
+            return this.simpleNewEntityName(newEntity.SchemaName, newEntity.TableName);
         }
     }
-    simpleNewEntityName(tableName) {
+    simpleNewEntityName(schemaName, tableName) {
         const convertedTableName = (0, global_1.convertCamelCaseToHaveSpaces)(tableName);
-        return (0, global_1.generatePluralName)(convertedTableName, { capitalizeFirstLetterOnly: true });
+        const pluralName = (0, global_1.generatePluralName)(convertedTableName, { capitalizeFirstLetterOnly: true });
+        return this.markupEntityName(schemaName, pluralName);
+    }
+    /**
+     * Uses the optional NameRulesBySchema section of the newEntityDefaults section of the config object to auto prefix/suffix a given entity name
+     * @param schemaName
+     * @param entityName
+     */
+    markupEntityName(schemaName, entityName) {
+        const rule = config_1.configInfo.newEntityDefaults?.NameRulesBySchema?.find(r => {
+            let schemaNameToUse = r.SchemaName;
+            if (schemaNameToUse?.trim().toLowerCase() === '${mj_core_schema}') {
+                // markup for this is to be replaced with the mj_core_schema() config
+                schemaNameToUse = (0, config_1.mj_core_schema)();
+            }
+            return schemaNameToUse.trim().toLowerCase() === schemaName.trim().toLowerCase();
+        });
+        if (rule) {
+            // found a matching rule, apply it
+            return rule.EntityNamePrefix + entityName + rule.EntityNameSuffix;
+        }
+        else {
+            // no matching rule, just return the entity name as is
+            return entityName;
+        }
     }
     createNewUUID() {
         return (0, uuid_1.v4)();
@@ -1669,7 +1754,7 @@ let ManageMetadataBase = ManageMetadataBase_1 = class ManageMetadataBase {
 exports.ManageMetadataBase = ManageMetadataBase;
 ManageMetadataBase._newEntityList = [];
 ManageMetadataBase._modifiedEntityList = [];
-ManageMetadataBase._generatedFieldValidators = [];
+ManageMetadataBase._generatedValidators = [];
 exports.ManageMetadataBase = ManageMetadataBase = ManageMetadataBase_1 = __decorate([
     (0, global_1.RegisterClass)(ManageMetadataBase)
 ], ManageMetadataBase);