@memberjunction/codegen-lib 2.48.0 → 2.50.0

package/dist/Database/manage-metadata.js

@@ -27,6 +27,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ManageMetadataBase = exports.ValidatorResult = void 0;
+const sql = __importStar(require("mssql"));
 const config_1 = require("../Config/config");
 const core_1 = require("@memberjunction/core");
 const status_logging_1 = require("../Misc/status_logging");
@@ -39,23 +40,22 @@ const path_1 = __importDefault(require("path"));
 const sql_logging_1 = require("../Misc/sql_logging");
 const aiengine_1 = require("@memberjunction/aiengine");
 class ValidatorResult {
-    constructor() {
-        this.entityName = "";
-        this.sourceCheckConstraint = "";
-        this.functionText = "";
-        this.functionName = "";
-        this.functionDescription = "";
-        /**
-         * The ID value in the Generated Codes entity that was created for this validator.
-         */
-        this.generatedCodeId = "";
-        /**
-         * The ID for the AI Model that was used to generate the code
-         */
-        this.aiModelID = "";
-        this.wasGenerated = true;
-        this.success = false;
-    }
+    entityName = "";
+    fieldName;
+    sourceCheckConstraint = "";
+    functionText = "";
+    functionName = "";
+    functionDescription = "";
+    /**
+     * The ID value in the Generated Codes entity that was created for this validator.
+     */
+    generatedCodeId = "";
+    /**
+     * The ID for the AI Model that was used to generate the code
+     */
+    aiModelID = "";
+    wasGenerated = true;
+    success = false;
 }
 exports.ValidatorResult = ValidatorResult;
 /**
@@ -63,24 +63,25 @@ exports.ValidatorResult = ValidatorResult;
  * to properly register your subclass with a priority of 1+ to ensure it gets instantiated.
  */
 class ManageMetadataBase {
-    constructor() {
-        this._sqlUtilityObject = global_1.MJGlobal.Instance.ClassFactory.CreateInstance(sql_1.SQLUtilityBase);
-    }
+    _sqlUtilityObject = global_1.MJGlobal.Instance.ClassFactory.CreateInstance(sql_1.SQLUtilityBase);
     get SQLUtilityObject() {
         return this._sqlUtilityObject;
     }
+    static _newEntityList = [];
     /**
      * Globally scoped list of entities that have been created during the metadata management process.
      */
     static get newEntityList() {
         return this._newEntityList;
     }
+    static _modifiedEntityList = [];
     /**
      * Globally scoped list of entities that have been modified during the metadata management process.
      */
     static get modifiedEntityList() {
         return this._modifiedEntityList;
     }
+    static _generatedValidators = [];
     /**
      * Globally scoped list of validators that have been generated during the metadata management process.
      */
@@ -89,30 +90,30 @@ class ManageMetadataBase {
     }
     /**
      * Primary function to manage metadata within the CodeGen system. This function will call a series of sub-functions to manage the metadata.
-     * @param ds - the DataSource object to use for querying and updating the database
+     * @param pool - the ConnectionPool object to use for querying and updating the database
      * @returns
      */
-    async manageMetadata(ds, currentUser) {
+    async manageMetadata(pool, currentUser) {
         const md = new core_1.Metadata();
         const excludeSchemas = config_1.configInfo.excludeSchemas ? config_1.configInfo.excludeSchemas : [];
         let bSuccess = true;
         let start = new Date();
         (0, status_logging_1.logStatus)('   Creating new entities...');
-        if (!await this.createNewEntities(ds)) {
+        if (!await this.createNewEntities(pool)) {
             (0, status_logging_1.logError)('   Error creating new entities');
             bSuccess = false;
         }
         (0, status_logging_1.logStatus)(`    > Created new entities in ${(new Date().getTime() - start.getTime()) / 1000} seconds`);
         start = new Date();
         (0, status_logging_1.logStatus)('   Updating existing entities...');
-        if (!await this.updateExistingEntitiesFromSchema(ds, excludeSchemas)) {
+        if (!await this.updateExistingEntitiesFromSchema(pool, excludeSchemas)) {
             (0, status_logging_1.logError)('   Error updating existing entities');
             bSuccess = false;
         }
         (0, status_logging_1.logStatus)(`    > Updated existing entities in ${(new Date().getTime() - start.getTime()) / 1000} seconds`);
         start = new Date();
         (0, status_logging_1.logStatus)('   Scanning for tables that were deleted where entity metadata still exists...');
-        if (!await this.checkAndRemoveMetadataForDeletedTables(ds, excludeSchemas)) {
+        if (!await this.checkAndRemoveMetadataForDeletedTables(pool, excludeSchemas)) {
             (0, status_logging_1.logError)('   Error removing metadata for tables that were removed');
             bSuccess = false;
         }
@@ -124,7 +125,7 @@ class ManageMetadataBase {
         const schemasToExclude = (0, config_1.getSettingValue)('recompile_mj_views', true)
             ? excludeSchemas.filter((s) => s !== adminSchema)
             : excludeSchemas;
-        if (!await sqlUtility.recompileAllBaseViews(ds, schemasToExclude, true, ManageMetadataBase._newEntityList /*exclude the newly created entities from the above step the first time we run as those views don't exist yet*/)) {
+        if (!await sqlUtility.recompileAllBaseViews(pool, schemasToExclude, true, ManageMetadataBase._newEntityList /*exclude the newly created entities from the above step the first time we run as those views don't exist yet*/)) {
             (0, status_logging_1.logMessage)('   Warning: Non-Fatal error recompiling base views', core_1.SeverityType.Warning, false);
             // many times the former versions of base views will NOT succesfully recompile, so don't consider that scenario to be a
             // failure for this entire function
@@ -132,40 +133,41 @@ class ManageMetadataBase {
         (0, status_logging_1.logStatus)(`    > Recompiled base views in ${(new Date().getTime() - start.getTime()) / 1000} seconds`);
         start = new Date();
         (0, status_logging_1.logStatus)('   Managing entity fields...');
-        if (!await this.manageEntityFields(ds, excludeSchemas, false, false, currentUser)) {
+        if (!await this.manageEntityFields(pool, excludeSchemas, false, false, currentUser)) {
             (0, status_logging_1.logError)('   Error managing entity fields');
             bSuccess = false;
         }
         (0, status_logging_1.logStatus)(`    > Managed entity fields in ${(new Date().getTime() - start.getTime()) / 1000} seconds`);
         start = new Date();
         (0, status_logging_1.logStatus)('   Managing entity relationships...');
-        if (!await this.manageEntityRelationships(ds, excludeSchemas, md)) {
+        if (!await this.manageEntityRelationships(pool, excludeSchemas, md)) {
             (0, status_logging_1.logError)('   Error managing entity relationships');
             bSuccess = false;
         }
         (0, status_logging_1.logStatus)(`    > Managed entity relationships in ${(new Date().getTime() - start.getTime()) / 1000} seconds`);
         if (ManageMetadataBase.newEntityList.length > 0) {
-            await this.generateNewEntityDescriptions(ds, md); // don't pass excludeSchemas becuase by definition this is the NEW entities we created
+            await this.generateNewEntityDescriptions(pool, md); // don't pass excludeSchemas becuase by definition this is the NEW entities we created
         }
-        const veResult = await this.manageVirtualEntities(ds);
+        const veResult = await this.manageVirtualEntities(pool);
         if (!veResult.success) {
             (0, status_logging_1.logError)('   Error managing virtual entities');
             bSuccess = false;
         }
         return bSuccess;
     }
-    async manageVirtualEntities(ds) {
+    async manageVirtualEntities(pool) {
         let bSuccess = true;
         // virtual entities are records defined in the entity metadata and do NOT define a distinct base table
         // but they do specify a base view. We DO NOT generate a base view for a virtual entity, we simply use it to figure
         // out the fields that should be in the entity definition and add/update/delete the entity definition to match what's in the view when this runs
         const sql = `SELECT * FROM [${(0, config_1.mj_core_schema)()}].vwEntities WHERE VirtualEntity = 1`;
-        const virtualEntities = await ds.query(sql);
+        const virtualEntitiesResult = await pool.request().query(sql);
+        const virtualEntities = virtualEntitiesResult.recordset;
         let anyUpdates = false;
         if (virtualEntities && virtualEntities.length > 0) {
             // we have 1+ virtual entities, now loop through them and process each one
             for (const ve of virtualEntities) {
-                const { success, updatedEntity } = await this.manageSingleVirtualEntity(ds, ve);
+                const { success, updatedEntity } = await this.manageSingleVirtualEntity(pool, ve);
                 anyUpdates = anyUpdates || updatedEntity;
                 if (!success) {
                     (0, status_logging_1.logError)(`   Error managing virtual entity ${ve.Name}`);
@@ -175,7 +177,7 @@ class ManageMetadataBase {
         }
         return { success: bSuccess, anyUpdates: anyUpdates };
     }
-    async manageSingleVirtualEntity(ds, virtualEntity) {
+    async manageSingleVirtualEntity(pool, virtualEntity) {
         let bSuccess = true;
         let bUpdated = false;
         try {
@@ -194,7 +196,8 @@ class ManageMetadataBase {
                            SCHEMA_NAME(v.schema_id) = '${virtualEntity.SchemaName}'
                         ORDER BY
                            c.column_id`;
-            const veFields = await ds.query(sql);
+            const veFieldsResult = await pool.request().query(sql);
+            const veFields = veFieldsResult.recordset;
             if (veFields && veFields.length > 0) {
                 // we have 1+ fields, now loop through them and process each one
                 // first though, remove any fields that are no longer in the view
@@ -209,7 +212,7 @@ class ManageMetadataBase {
                     if (removeList.length > 0) {
                         const sqlRemove = `DELETE FROM [${(0, config_1.mj_core_schema)()}].EntityField WHERE ID IN (${removeList.map(removeId => `'${removeId}'`).join(',')})`;
                         // this removes the fields that shouldn't be there anymore
-                        await this.LogSQLAndExecute(ds, sqlRemove, `SQL text to remove fields from entity ${virtualEntity.Name}`);
+                        await this.LogSQLAndExecute(pool, sqlRemove, `SQL text to remove fields from entity ${virtualEntity.Name}`);
                         bUpdated = true;
                     }
                     // check to see if any of the fields in the virtual entity have Pkey attribute set. If not, we will default to the first field
@@ -218,7 +221,7 @@ class ManageMetadataBase {
                     // now create/update the fields that are in the view
                     for (let i = 0; i < veFields.length; i++) {
                         const vef = veFields[i];
-                        const { success, updatedField } = await this.manageSingleVirtualEntityField(ds, virtualEntity, vef, i + 1, !hasPkey && i === 0);
+                        const { success, updatedField } = await this.manageSingleVirtualEntityField(pool, virtualEntity, vef, i + 1, !hasPkey && i === 0);
                         bUpdated = bUpdated || updatedField;
                         if (!success) {
                             (0, status_logging_1.logError)(`Error managing virtual entity field ${vef.FieldName} for virtual entity ${virtualEntity.Name}`);
@@ -230,7 +233,7 @@ class ManageMetadataBase {
             if (bUpdated) {
                 // finally make sure we update the UpdatedAt field for the entity if we made changes to its fields
                 const sqlUpdate = `UPDATE [${(0, config_1.mj_core_schema)()}].Entity SET [${core_1.EntityInfo.UpdatedAtFieldName}]=GETUTCDATE() WHERE ID='${virtualEntity.ID}'`;
-                await this.LogSQLAndExecute(ds, sqlUpdate, `SQL text to update virtual entity updated date for ${virtualEntity.Name}`);
+                await this.LogSQLAndExecute(pool, sqlUpdate, `SQL text to update virtual entity updated date for ${virtualEntity.Name}`);
             }
             return { success: bSuccess, updatedEntity: bUpdated };
         }
@@ -239,7 +242,7 @@ class ManageMetadataBase {
             return { success: false, updatedEntity: bUpdated };
         }
     }
-    async manageSingleVirtualEntityField(ds, virtualEntity, veField, fieldSequence, makePrimaryKey) {
+    async manageSingleVirtualEntityField(pool, virtualEntity, veField, fieldSequence, makePrimaryKey) {
         // this protected checks to see if the field exists in the entity definition, and if not, adds it
         // if it exist it updates the entity field to match the view's data type and nullability attributes
         // first, get the entity definition
@@ -271,7 +274,7 @@ class ManageMetadataBase {
                                     Scale=${veField.Scale}
                                   WHERE
                                     ID = '${field.ID}'`; // don't need to update the __mj_UpdatedAt field here, that happens automatically via the trigger
-                    await this.LogSQLAndExecute(ds, sqlUpdate, `SQL text to update virtual entity field ${veField.FieldName} for entity ${virtualEntity.Name}`);
+                    await this.LogSQLAndExecute(pool, sqlUpdate, `SQL text to update virtual entity field ${veField.FieldName} for entity ${virtualEntity.Name}`);
                     didUpdate = true;
                 }
             }
@@ -286,7 +289,7 @@ class ManageMetadataBase {
                                        ${veField.Length}, ${veField.Precision}, ${veField.Scale},
                                        ${fieldSequence}, ${makePrimaryKey ? 1 : 0}, ${makePrimaryKey ? 1 : 0}
                                     )`;
-                await this.LogSQLAndExecute(ds, sqlAdd, `SQL text to add virtual entity field ${veField.FieldName} for entity ${virtualEntity.Name}`);
+                await this.LogSQLAndExecute(pool, sqlAdd, `SQL text to add virtual entity field ${veField.FieldName} for entity ${virtualEntity.Name}`);
                 didUpdate = true;
             }
         }
@@ -294,25 +297,25 @@ class ManageMetadataBase {
     }
     /**
      * This method creates and updates relationships in the metadata based on foreign key relationships in the database.
-     * @param ds
+     * @param pool
      * @param excludeSchemas - specify any schemas to exclude here and any relationships to/from the specified schemas will be ignored
      * @param md
      * @returns
      */
-    async manageEntityRelationships(ds, excludeSchemas, md, batchItems = 5) {
+    async manageEntityRelationships(pool, excludeSchemas, md, batchItems = 5) {
         let bResult = true;
-        bResult = bResult && await this.manageManyToManyEntityRelationships(ds, excludeSchemas, batchItems);
-        bResult = bResult && await this.manageOneToManyEntityRelationships(ds, excludeSchemas, md, batchItems);
+        bResult = bResult && await this.manageManyToManyEntityRelationships(pool, excludeSchemas, batchItems);
+        bResult = bResult && await this.manageOneToManyEntityRelationships(pool, excludeSchemas, md, batchItems);
         return bResult;
     }
     /**
      * Manages 1->M relationships between entities in the metadata based on foreign key relationships in the database.
-     * @param ds
+     * @param pool
      * @param excludeSchemas - specify any schemas to exclude here and any relationships to/from the specified schemas will be ignored
      * @param md
      * @returns
      */
-    async manageOneToManyEntityRelationships(ds, excludeSchemas, md, batchItems = 5) {
+    async manageOneToManyEntityRelationships(pool, excludeSchemas, md, batchItems = 5) {
         // the way this works is that we look for entities in our catalog and we look for
         // foreign keys in those entities. For example, if we saw an entity called Persons and that entity
         // had a foreign key linking to an entity called Organizations via a field called OrganizationID, then we would create a relationship
@@ -334,17 +337,20 @@ class ManageMetadataBase {
                              IsVirtual = 0 AND
                              EntityID NOT IN (SELECT ID FROM ${(0, config_1.mj_core_schema)()}.Entity WHERE SchemaName IN (${excludeSchemas.map(s => `'${s}'`).join(',')}))
                        ORDER BY RelatedEntityID`;
-            const entityFields = await ds.query(sSQL);
+            const entityFieldsResult = await pool.request().query(sSQL);
+            const entityFields = entityFieldsResult.recordset;
             // Get the relationship counts for each entity
             const sSQLRelationshipCount = `SELECT EntityID, COUNT(*) AS Count FROM ${(0, config_1.mj_core_schema)()}.EntityRelationship GROUP BY EntityID`;
-            const relationshipCounts = await ds.query(sSQLRelationshipCount);
+            const relationshipCountsResult = await pool.request().query(sSQLRelationshipCount);
+            const relationshipCounts = relationshipCountsResult.recordset;
             const relationshipCountMap = new Map();
             for (const rc of relationshipCounts) {
                 relationshipCountMap.set(rc.EntityID, rc.Count);
             }
             // get all relationships in one query for performance improvement
             const sSQLRelationship = `SELECT * FROM ${(0, config_1.mj_core_schema)()}.EntityRelationship`;
-            const allRelationships = await ds.query(sSQLRelationship);
+            const allRelationshipsResult = await pool.request().query(sSQLRelationship);
+            const allRelationships = allRelationshipsResult.recordset;
             // Function to process a batch of entity fields
             const processBatch = async (batch) => {
                 let batchSQL = '';
@@ -374,7 +380,7 @@ class ManageMetadataBase {
                     }
                 });
                 if (batchSQL.length > 0) {
-                    await this.LogSQLAndExecute(ds, batchSQL, `SQL text to create Entitiy Relationships`);
+                    await this.LogSQLAndExecute(pool, batchSQL, `SQL text to create Entitiy Relationships`);
                 }
             };
             // Split entityFields into batches and process each batch
@@ -395,10 +401,11 @@ class ManageMetadataBase {
      * @param ds
      * @param excludeSchemas
      */
-    async checkAndRemoveMetadataForDeletedTables(ds, excludeSchemas) {
+    async checkAndRemoveMetadataForDeletedTables(pool, excludeSchemas) {
         try {
             const sql = `SELECT * FROM ${(0, config_1.mj_core_schema)()}.vwEntitiesWithMissingBaseTables WHERE VirtualEntity=0`;
-            const entities = await ds.query(sql);
+            const entitiesResult = await pool.request().query(sql);
+            const entities = entitiesResult.recordset;
             if (entities && entities.length > 0) {
                 for (const e of entities) {
                     // for the given entity, wipe out the entity metadata and its core deps.
@@ -406,15 +413,15 @@ class ManageMetadataBase {
                     // for the admin to handle manually
                     try {
                         const sqlDelete = `__mj.spDeleteEntityWithCoreDependencies @EntityID='${e.ID}'`;
-                        await this.LogSQLAndExecute(ds, sqlDelete, `SQL text to remove entity ${e.Name}`);
+                        await this.LogSQLAndExecute(pool, sqlDelete, `SQL text to remove entity ${e.Name}`);
                         (0, status_logging_1.logStatus)(`      > Removed metadata for table ${e.SchemaName}.${e.BaseTable}`);
                         // next up we need to remove the spCreate, spDelete, spUpdate, BaseView, and FullTextSearchFunction, if provided.
                         // We only remoe these artifcacts when they are generated which is info we have in the BaseViewGenerated, spCreateGenerated, etc. fields
-                        await this.checkDropSQLObject(ds, e.BaseViewGenerated, 'view', e.SchemaName, e.BaseView);
-                        await this.checkDropSQLObject(ds, e.spCreateGenerated, 'procedure', e.SchemaName, e.spCreate ? e.spCreate : `spCreate${e.ClassName}`);
-                        await this.checkDropSQLObject(ds, e.spDeleteGenerated, 'procedure', e.SchemaName, e.spDelete ? e.spDelete : `spDelete${e.ClassName}`);
-                        await this.checkDropSQLObject(ds, e.spUpdateGenerated, 'procedure', e.SchemaName, e.spUpdate ? e.spUpdate : `spUpdate${e.ClassName}`);
-                        await this.checkDropSQLObject(ds, e.FullTextSearchFunctionGenerated, 'function', e.SchemaName, e.FullTextSearchFunction);
+                        await this.checkDropSQLObject(pool, e.BaseViewGenerated, 'view', e.SchemaName, e.BaseView);
+                        await this.checkDropSQLObject(pool, e.spCreateGenerated, 'procedure', e.SchemaName, e.spCreate ? e.spCreate : `spCreate${e.ClassName}`);
+                        await this.checkDropSQLObject(pool, e.spDeleteGenerated, 'procedure', e.SchemaName, e.spDelete ? e.spDelete : `spDelete${e.ClassName}`);
+                        await this.checkDropSQLObject(pool, e.spUpdateGenerated, 'procedure', e.SchemaName, e.spUpdate ? e.spUpdate : `spUpdate${e.ClassName}`);
+                        await this.checkDropSQLObject(pool, e.FullTextSearchFunctionGenerated, 'function', e.SchemaName, e.FullTextSearchFunction);
                     }
                     catch (ex) {
                         (0, status_logging_1.logError)(`Error removing metadata for entity ${ex.Name}, error: ${ex}`);
@@ -431,11 +438,11 @@ class ManageMetadataBase {
             return false;
         }
     }
-    async checkDropSQLObject(ds, proceed, type, schemaName, name) {
+    async checkDropSQLObject(pool, proceed, type, schemaName, name) {
         try {
             if (proceed && schemaName && name && schemaName.trim().length > 0 && name.trim().length > 0) {
                 const sqlDelete = `DROP ${type} IF EXISTS [${schemaName}].[${name}]`;
-                await this.LogSQLAndExecute(ds, sqlDelete, `SQL text to remove ${type} ${schemaName}.${name}`);
+                await this.LogSQLAndExecute(pool, sqlDelete, `SQL text to remove ${type} ${schemaName}.${name}`);
                 // next up, we need to clean up the cache of saved DB objects that may exist for this entity in the appropriate sub-directory.
                 const sqlOutputDir = (0, config_1.outputDir)('SQL', true);
                 if (sqlOutputDir) {
@@ -464,52 +471,58 @@ class ManageMetadataBase {
      * @param excludeSchemas
      * @returns
      */
-    async manageManyToManyEntityRelationships(ds, excludeSchemas, batchItems = 5) {
+    async manageManyToManyEntityRelationships(pool, excludeSchemas, batchItems = 5) {
         return true; // not implemented for now, require the admin to manually create these relationships
     }
     /**
      * Manages the creation, updating and deletion of entity field records in the metadata based on the database schema.
-     * @param ds
+     * @param pool
      * @param excludeSchemas
      * @returns
      */
-    async manageEntityFields(ds, excludeSchemas, skipCreatedAtUpdatedAtDeletedAtFieldValidation, skipEntityFieldValues, currentUser) {
+    async manageEntityFields(pool, excludeSchemas, skipCreatedAtUpdatedAtDeletedAtFieldValidation, skipEntityFieldValues, currentUser) {
         let bSuccess = true;
         const startTime = new Date();
         if (!skipCreatedAtUpdatedAtDeletedAtFieldValidation) {
-            if (!await this.ensureCreatedAtUpdatedAtFieldsExist(ds, excludeSchemas) ||
-                !await this.ensureDeletedAtFieldsExist(ds, excludeSchemas)) {
+            if (!await this.ensureCreatedAtUpdatedAtFieldsExist(pool, excludeSchemas) ||
+                !await this.ensureDeletedAtFieldsExist(pool, excludeSchemas)) {
                 (0, status_logging_1.logError)(`Error ensuring ${core_1.EntityInfo.CreatedAtFieldName}, ${core_1.EntityInfo.UpdatedAtFieldName} and ${core_1.EntityInfo.DeletedAtFieldName} fields exist`);
                 bSuccess = false;
             }
             (0, status_logging_1.logStatus)(`      Ensured ${core_1.EntityInfo.CreatedAtFieldName}/${core_1.EntityInfo.UpdatedAtFieldName}/${core_1.EntityInfo.DeletedAtFieldName} fields exist in ${(new Date().getTime() - startTime.getTime()) / 1000} seconds`);
         }
         const step1StartTime = new Date();
-        if (!await this.deleteUnneededEntityFields(ds, excludeSchemas)) {
+        if (!await this.deleteUnneededEntityFields(pool, excludeSchemas)) {
             (0, status_logging_1.logError)('Error deleting unneeded entity fields');
             bSuccess = false;
         }
         (0, status_logging_1.logStatus)(`      Deleted unneeded entity fields in ${(new Date().getTime() - step1StartTime.getTime()) / 1000} seconds`);
+        // AN: 14-June-2025 - See note below about the new order of these steps, this must
+        // happen before we update existing entity fields from schema.
         const step2StartTime = new Date();
-        if (!await this.updateExistingEntityFieldsFromSchema(ds, excludeSchemas)) {
-            (0, status_logging_1.logError)('Error updating existing entity fields from schema');
+        if (!await this.createNewEntityFieldsFromSchema(pool)) { // has its own internal filtering for exclude schema/table so don't pass in
+            (0, status_logging_1.logError)('Error creating new entity fields from schema');
             bSuccess = false;
         }
-        (0, status_logging_1.logStatus)(`      Updated existing entity fields from schema in ${(new Date().getTime() - step2StartTime.getTime()) / 1000} seconds`);
+        (0, status_logging_1.logStatus)(`      Created new entity fields from schema in ${(new Date().getTime() - step2StartTime.getTime()) / 1000} seconds`);
+        // AN: 14-June-2025 - we are now running this AFTER we create new entity fields from schema
+        // which results in the same pattern of behavior as migrations where we first create new fields
+        // with VERY HIGH sequence numbers (e.g. 100,000 above what they will be approx) and then
+        // we align them properly in sequential order from 1+ via this method below.
         const step3StartTime = new Date();
-        if (!await this.createNewEntityFieldsFromSchema(ds)) { // has its own internal filtering for exclude schema/table so don't pass in
-            (0, status_logging_1.logError)('Error creating new entity fields from schema');
+        if (!await this.updateExistingEntityFieldsFromSchema(pool, excludeSchemas)) {
+            (0, status_logging_1.logError)('Error updating existing entity fields from schema');
             bSuccess = false;
         }
-        (0, status_logging_1.logStatus)(`      Created new entity fields from schema in ${(new Date().getTime() - step3StartTime.getTime()) / 1000} seconds`);
+        (0, status_logging_1.logStatus)(`      Updated existing entity fields from schema in ${(new Date().getTime() - step3StartTime.getTime()) / 1000} seconds`);
         const step4StartTime = new Date();
-        if (!await this.setDefaultColumnWidthWhereNeeded(ds, excludeSchemas)) {
+        if (!await this.setDefaultColumnWidthWhereNeeded(pool, excludeSchemas)) {
             (0, status_logging_1.logError)('Error setting default column width where needed');
             bSuccess = false;
         }
         (0, status_logging_1.logStatus)(`      Set default column width where needed in ${(new Date().getTime() - step4StartTime.getTime()) / 1000} seconds`);
         const step5StartTime = new Date();
-        if (!await this.updateEntityFieldDisplayNameWhereNull(ds, excludeSchemas)) {
+        if (!await this.updateEntityFieldDisplayNameWhereNull(pool, excludeSchemas)) {
             (0, status_logging_1.logError)('Error updating entity field display name where null');
             bSuccess = false;
         }
@@ -517,7 +530,7 @@ class ManageMetadataBase {
         if (!skipEntityFieldValues) {
             const step6StartTime = new Date();
             (0, status_logging_1.logStatus)(`      Starting to manage entity field values...`);
-            if (!await this.manageEntityFieldValuesAndValidatorFunctions(ds, excludeSchemas, currentUser, false)) {
+            if (!await this.manageEntityFieldValuesAndValidatorFunctions(pool, excludeSchemas, currentUser, false)) {
                 (0, status_logging_1.logError)('Error managing entity field values');
                 bSuccess = false;
             }
@@ -529,7 +542,7 @@ class ManageMetadataBase {
     /**
      * This method ensures that the __mj_DeletedAt field exists in each entity that has DeleteType=Soft. If the field does not exist, it is created.
      */
-    async ensureDeletedAtFieldsExist(ds, excludeSchemas) {
+    async ensureDeletedAtFieldsExist(pool, excludeSchemas) {
         try {
             const sqlEntities = `SELECT
                                  *
@@ -539,7 +552,8 @@ class ManageMetadataBase {
                                  VirtualEntity=0 AND
                                  DeleteType='Soft' AND
                                  SchemaName NOT IN (${excludeSchemas.map(s => `'${s}'`).join(',')})`;
-            const entities = await ds.query(sqlEntities);
+            const entitiesResult = await pool.request().query(sqlEntities);
+            const entities = entitiesResult.recordset;
             let overallResult = true;
             if (entities.length > 0) {
                 // we have 1+ entities that need the special fields, so loop through them and ensure the fields exist
@@ -549,12 +563,13 @@ class ManageMetadataBase {
                          WHERE
                          ${entities.map((e) => `(TABLE_SCHEMA='${e.SchemaName}' AND TABLE_NAME='${e.BaseTable}')`).join(' OR ')}
                          AND COLUMN_NAME='${core_1.EntityInfo.DeletedAtFieldName}'`;
-                const result = await ds.query(sql);
+                const resultResult = await pool.request().query(sql);
+                const result = resultResult.recordset;
                 for (const e of entities) {
                     const eResult = result.filter((r) => r.TABLE_NAME === e.BaseTable && r.TABLE_SCHEMA === e.SchemaName); // get just the fields for this entity
                     const deletedAt = eResult.find((r) => r.COLUMN_NAME.trim().toLowerCase() === core_1.EntityInfo.DeletedAtFieldName.trim().toLowerCase());
                     // now, if we have the fields, we need to check the default value and update if necessary
-                    const fieldResult = await this.ensureSpecialDateFieldExistsAndHasCorrectDefaultValue(ds, e, core_1.EntityInfo.DeletedAtFieldName, deletedAt, true);
+                    const fieldResult = await this.ensureSpecialDateFieldExistsAndHasCorrectDefaultValue(pool, e, core_1.EntityInfo.DeletedAtFieldName, deletedAt, true);
                     overallResult = overallResult && fieldResult;
                 }
             }
@@ -569,9 +584,9 @@ class ManageMetadataBase {
      * This method ensures that the __mj_CreatedAt and __mj_UpdatedAt fields exist in each entity that has TrackRecordChanges set to true. If the fields do not exist, they are created.
      * If the fields exist but have incorrect default values, the default values are updated. The default value that is to be used for these special fields is GETUTCDATE() which is the
      * UTC date and time. This method is called as part of the manageEntityFields method and is not intended to be called directly.
-     * @param ds
+     * @param pool
      */
-    async ensureCreatedAtUpdatedAtFieldsExist(ds, excludeSchemas) {
+    async ensureCreatedAtUpdatedAtFieldsExist(pool, excludeSchemas) {
         try {
             const sqlEntities = `SELECT
                                  *
@@ -581,7 +596,8 @@ class ManageMetadataBase {
                                  VirtualEntity = 0 AND
                                  TrackRecordChanges = 1 AND
                                  SchemaName NOT IN (${excludeSchemas.map(s => `'${s}'`).join(',')})`;
-            const entities = await ds.query(sqlEntities);
+            const entitiesResult = await pool.request().query(sqlEntities);
+            const entities = entitiesResult.recordset;
             let overallResult = true;
             if (entities.length > 0) {
                 // we have 1+ entities that need the special fields, so loop through them and ensure the fields exist
@@ -592,15 +608,16 @@ class ManageMetadataBase {
                                        WHERE
                                           ${entities.map((e) => `(TABLE_SCHEMA='${e.SchemaName}' AND TABLE_NAME='${e.BaseTable}')`).join(' OR ')}
                                        AND COLUMN_NAME IN ('${core_1.EntityInfo.CreatedAtFieldName}','${core_1.EntityInfo.UpdatedAtFieldName}')`;
-                const result = await ds.query(sqlCreatedUpdated);
+                const resultResult = await pool.request().query(sqlCreatedUpdated);
+                const result = resultResult.recordset;
                 for (const e of entities) {
                     // result has both created at and updated at fields, so filter on the result for each and do what we need to based on that
                     const eResult = result.filter((r) => r.TABLE_NAME === e.BaseTable && r.TABLE_SCHEMA === e.SchemaName); // get just the fields for this entity
                     const createdAt = eResult.find((r) => r.COLUMN_NAME.trim().toLowerCase() === core_1.EntityInfo.CreatedAtFieldName.trim().toLowerCase());
                     const updatedAt = eResult.find((r) => r.COLUMN_NAME.trim().toLowerCase() === core_1.EntityInfo.UpdatedAtFieldName.trim().toLowerCase());
                     // now, if we have the fields, we need to check the default value and update if necessary
-                    const fieldResult = await this.ensureSpecialDateFieldExistsAndHasCorrectDefaultValue(ds, e, core_1.EntityInfo.CreatedAtFieldName, createdAt, false) &&
-                        await this.ensureSpecialDateFieldExistsAndHasCorrectDefaultValue(ds, e, core_1.EntityInfo.UpdatedAtFieldName, updatedAt, false);
+                    const fieldResult = await this.ensureSpecialDateFieldExistsAndHasCorrectDefaultValue(pool, e, core_1.EntityInfo.CreatedAtFieldName, createdAt, false) &&
+                        await this.ensureSpecialDateFieldExistsAndHasCorrectDefaultValue(pool, e, core_1.EntityInfo.UpdatedAtFieldName, updatedAt, false);
                     overallResult = overallResult && fieldResult;
                 }
             }
@@ -618,12 +635,12 @@ class ManageMetadataBase {
      * @param fieldName
      * @param currentFieldData
      */
-    async ensureSpecialDateFieldExistsAndHasCorrectDefaultValue(ds, entity, fieldName, currentFieldData, allowNull) {
+    async ensureSpecialDateFieldExistsAndHasCorrectDefaultValue(pool, entity, fieldName, currentFieldData, allowNull) {
         try {
             if (!currentFieldData) {
                 // field doesn't exist, let's create it
                 const sql = `ALTER TABLE [${entity.SchemaName}].[${entity.BaseTable}] ADD ${fieldName} DATETIMEOFFSET ${allowNull ? 'NULL' : 'NOT NULL DEFAULT GETUTCDATE()'}`;
-                await this.LogSQLAndExecute(ds, sql, `SQL text to add special date field ${fieldName} to entity ${entity.SchemaName}.${entity.BaseTable}`);
+                await this.LogSQLAndExecute(pool, sql, `SQL text to add special date field ${fieldName} to entity ${entity.SchemaName}.${entity.BaseTable}`);
             }
             else {
                 // field does exist, let's first check the data type/nullability
@@ -632,11 +649,11 @@ class ManageMetadataBase {
                     (currentFieldData.IS_NULLABLE.trim().toLowerCase() === 'no' && allowNull)) {
                     // the column is the wrong type, or has wrong nullability attribute, so let's update it, first removing the default constraint, then
                     // modifying the column, and finally adding the default constraint back in.
-                    await this.dropExistingDefaultConstraint(ds, entity, fieldName);
+                    await this.dropExistingDefaultConstraint(pool, entity, fieldName);
                     const sql = `ALTER TABLE [${entity.SchemaName}].[${entity.BaseTable}] ALTER COLUMN ${fieldName} DATETIMEOFFSET ${allowNull ? 'NULL' : 'NOT NULL'}`;
-                    await this.LogSQLAndExecute(ds, sql, `SQL text to update special date field ${fieldName} in entity ${entity.SchemaName}.${entity.BaseTable}`);
+                    await this.LogSQLAndExecute(pool, sql, `SQL text to update special date field ${fieldName} in entity ${entity.SchemaName}.${entity.BaseTable}`);
                     if (!allowNull)
-                        await this.createDefaultConstraintForSpecialDateField(ds, entity, fieldName);
+                        await this.createDefaultConstraintForSpecialDateField(pool, entity, fieldName);
                 }
                 else {
                     // if we get here that means the column is the correct type and nullability, so now let's check the default value, but we only do that if we are dealing with a
@@ -645,7 +662,7 @@ class ManageMetadataBase {
                         const defaultValue = currentFieldData.COLUMN_DEFAULT;
                         const realDefaultValue = (0, core_1.ExtractActualDefaultValue)(defaultValue);
                         if (!realDefaultValue || realDefaultValue.trim().toLowerCase() !== 'getutcdate()') {
-                            await this.dropAndCreateDefaultConstraintForSpecialDateField(ds, entity, fieldName);
+                            await this.dropAndCreateDefaultConstraintForSpecialDateField(pool, entity, fieldName);
                         }
                     }
                 }
@@ -661,10 +678,10 @@ class ManageMetadataBase {
     /**
      * Creates the default constraint for a special date field. This method is called as part of the ensureSpecialDateFieldExistsAndHasCorrectDefaultValue method and is not intended to be called directly.
      */
-    async createDefaultConstraintForSpecialDateField(ds, entity, fieldName) {
+    async createDefaultConstraintForSpecialDateField(pool, entity, fieldName) {
         try {
             const sqlAddDefaultConstraint = `ALTER TABLE [${entity.SchemaName}].[${entity.BaseTable}] ADD CONSTRAINT DF_${entity.SchemaName}_${(0, core_1.CodeNameFromString)(entity.BaseTable)}_${fieldName} DEFAULT GETUTCDATE() FOR [${fieldName}]`;
-            await this.LogSQLAndExecute(ds, sqlAddDefaultConstraint, `SQL text to add default constraint for special date field ${fieldName} in entity ${entity.SchemaName}.${entity.BaseTable}`);
+            await this.LogSQLAndExecute(pool, sqlAddDefaultConstraint, `SQL text to add default constraint for special date field ${fieldName} in entity ${entity.SchemaName}.${entity.BaseTable}`);
         }
         catch (e) {
             (0, status_logging_1.logError)(e);
@@ -676,18 +693,18 @@ class ManageMetadataBase {
      * @param entity
      * @param fieldName
      */
-    async dropAndCreateDefaultConstraintForSpecialDateField(ds, entity, fieldName) {
+    async dropAndCreateDefaultConstraintForSpecialDateField(pool, entity, fieldName) {
         // default value is not correct, so let's update it
-        await this.dropExistingDefaultConstraint(ds, entity, fieldName);
-        await this.createDefaultConstraintForSpecialDateField(ds, entity, fieldName);
+        await this.dropExistingDefaultConstraint(pool, entity, fieldName);
+        await this.createDefaultConstraintForSpecialDateField(pool, entity, fieldName);
     }
     /**
      * Drops an existing default constraint from a given column within a given entity, if it exists
-     * @param ds
+     * @param pool
      * @param entity
      * @param fieldName
      */
-    async dropExistingDefaultConstraint(ds, entity, fieldName) {
+    async dropExistingDefaultConstraint(pool, entity, fieldName) {
         try {
             const sqlDropDefaultConstraint = `
          DECLARE @constraintName NVARCHAR(255);
@@ -708,7 +725,7 @@ class ManageMetadataBase {
             EXEC('ALTER TABLE [${entity.SchemaName}].[${entity.BaseTable}] DROP CONSTRAINT ' + @constraintName);
          END
          `;
-            await this.LogSQLAndExecute(ds, sqlDropDefaultConstraint, `SQL text to drop default existing default constraints in entity ${entity.SchemaName}.${entity.BaseTable}`);
+            await this.LogSQLAndExecute(pool, sqlDropDefaultConstraint, `SQL text to drop default existing default constraints in entity ${entity.SchemaName}.${entity.BaseTable}`);
         }
         catch (e) {
             (0, status_logging_1.logError)(e);
@@ -717,10 +734,10 @@ class ManageMetadataBase {
     /**
      * This method generates descriptions for entities in teh system where there is no existing description. This is an experimental feature and is done using AI. In order for it
      * to be invoked, the EntityDescriptions feature must be enabled in the Advanced Generation configuration.
-     * @param ds
+     * @param pool
      * @param md
      */
-    async generateNewEntityDescriptions(ds, md) {
+    async generateNewEntityDescriptions(pool, md) {
         // for the list of new entities, go through and attempt to generate new entity descriptions
         const ag = new advanced_generation_1.AdvancedGeneration();
         if (ag.featureEnabled('EntityDescriptions')) {
@@ -731,8 +748,10 @@ class ManageMetadataBase {
             const userMessage = prompt.userMessage + '\n\n';
             // now loop through the new entities and generate descriptions for them
             for (let e of ManageMetadataBase.newEntityList) {
-                const data = await ds.query(`SELECT * FROM [${(0, config_1.mj_core_schema)()}].vwEntities WHERE Name = '${e}'`);
-                const fields = await ds.query(`SELECT * FROM [${(0, config_1.mj_core_schema)()}].vwEntityFields WHERE EntityID='${data[0].ID}'`);
+                const dataResult = await pool.request().query(`SELECT * FROM [${(0, config_1.mj_core_schema)()}].vwEntities WHERE Name = '${e}'`);
+                const data = dataResult.recordset;
+                const fieldsResult = await pool.request().query(`SELECT * FROM [${(0, config_1.mj_core_schema)()}].vwEntityFields WHERE EntityID='${data[0].ID}'`);
+                const fields = fieldsResult.recordset;
                 const entityUserMessage = userMessage + `Entity Name: ${e},
                                                      Base Table: ${data[0].BaseTable},
                                                      Schema: ${data[0].SchemaName}.
@@ -757,7 +776,7 @@ class ManageMetadataBase {
                         const structuredResult = JSON.parse(resultText);
                         if (structuredResult?.entityDescription && structuredResult.entityDescription.length > 0) {
                             const sSQL = `UPDATE [${(0, config_1.mj_core_schema)()}].Entity SET Description = '${structuredResult.entityDescription}' WHERE Name = '${e}'`;
-                            await this.LogSQLAndExecute(ds, sSQL, `SQL text to update entity description for entity ${e}`);
+                            await this.LogSQLAndExecute(pool, sSQL, `SQL text to update entity description for entity ${e}`);
                         }
                         else {
                             console.warn('   >>> Advanced Generation Error: LLM returned a blank entity description, skipping entity description for entity ' + e);
@@ -781,7 +800,7 @@ class ManageMetadataBase {
      * @param excludeSchemas
      * @returns
      */
-    async updateEntityFieldDisplayNameWhereNull(ds, excludeSchemas) {
+    async updateEntityFieldDisplayNameWhereNull(pool, excludeSchemas) {
         try {
             const sql = `SELECT
                         ef.ID, ef.Name
@@ -797,13 +816,14 @@ class ManageMetadataBase {
                         ef.Name <> \'ID\' AND
                         e.SchemaName NOT IN (${excludeSchemas.map(s => `'${s}'`).join(',')})
                         `;
-            const fields = await ds.query(sql);
+            const fieldsResult = await pool.request().query(sql);
+            const fields = fieldsResult.recordset;
             if (fields && fields.length > 0)
                 for (const field of fields) {
                     const sDisplayName = (0, global_1.stripTrailingChars)((0, global_1.convertCamelCaseToHaveSpaces)(field.Name), 'ID', true).trim();
                     if (sDisplayName.length > 0 && sDisplayName.toLowerCase().trim() !== field.Name.toLowerCase().trim()) {
                         const sSQL = `UPDATE [${(0, config_1.mj_core_schema)()}].EntityField SET ${core_1.EntityInfo.UpdatedAtFieldName}=GETUTCDATE(), DisplayName = '${sDisplayName}' WHERE ID = '${field.ID}'`;
-                        await this.LogSQLAndExecute(ds, sSQL, `SQL text to update display name for field ${field.Name}`);
+                        await this.LogSQLAndExecute(pool, sSQL, `SQL text to update display name for field ${field.Name}`);
                     }
                 }
             return true;
@@ -817,14 +837,14 @@ class ManageMetadataBase {
      * This method updates the DefaultColumnWidth field in the EntityField metadata. The default logic uses a stored procedure called spSetDefaultColumnWidthWhereNeeded
      * which is part of the MJ Core Schema. You can override this method to implement custom logic for setting default column widths. It is NOT recommended to
      * modify the stored procedure in the MJ Core Schema because your changes will be overriden during a future upgrade.
-     * @param ds
+     * @param pool
      * @param excludeSchemas
      * @returns
      */
-    async setDefaultColumnWidthWhereNeeded(ds, excludeSchemas) {
+    async setDefaultColumnWidthWhereNeeded(pool, excludeSchemas) {
         try {
             const sSQL = `EXEC ${(0, config_1.mj_core_schema)()}.spSetDefaultColumnWidthWhereNeeded @ExcludedSchemaNames='${excludeSchemas.join(',')}'`;
-            await this.LogSQLAndExecute(ds, sSQL, `SQL text to set default column width where needed`, true);
+            await this.LogSQLAndExecute(pool, sSQL, `SQL text to set default column width where needed`, true);
             return true;
         }
         catch (e) {
@@ -835,12 +855,18 @@ class ManageMetadataBase {
     /**
      * Creates a SQL statement to retrieve all of the pending entity fields that need to be created in the metadata. This method looks for fields that exist in the underlying
      * database but are NOT in the metadata.
+     *
+     * IMPORTANT: The sequence shown below has a 100,000 added to it to ensure that there is no collision with existing sequences. The spUpdateExistingEntityFieldsFromSchema
+     * stored procedure runs AFTER this method and will correct the sequences to ensure they are in the correct order. In a migration, the spUpdateExistingEntityFieldsFromSchema
+     * runs afterwards as well so this behavior ensures CodeGen works consistently.
+     *
+     * @returns {string} - The SQL statement to retrieve pending entity fields.
      */
     getPendingEntityFieldsSELECTSQL() {
         const sSQL = `WITH NumberedRows AS (
    SELECT
       sf.EntityID,
-      sf.Sequence,
+      sf.Sequence + 100000 Sequence, -- add a large number to the sequence to ensure no collision with existing sequences - spUpdateExistingEntityFieldsFromSchema runs AFTER this and will correct them.
       sf.FieldName,
       sf.Description,
       sf.Type,
@@ -1044,11 +1070,14 @@ class ManageMetadataBase {
         }
         return sResult;
     }
-    async createNewEntityFieldsFromSchema(ds) {
+    async createNewEntityFieldsFromSchema(pool) {
         try {
             const sSQL = this.getPendingEntityFieldsSELECTSQL();
-            const newEntityFields = await ds.query(sSQL);
-            await ds.transaction(async () => {
+            const newEntityFieldsResult = await pool.request().query(sSQL);
+            const newEntityFields = newEntityFieldsResult.recordset;
+            const transaction = new sql.Transaction(pool);
+            await transaction.begin();
+            try {
                 // wrap in a transaction so we get all of it or none of it
                 for (let i = 0; i < newEntityFields.length; ++i) {
                     const n = newEntityFields[i];
@@ -1058,7 +1087,7 @@ class ManageMetadataBase {
                         const newEntityFieldUUID = this.createNewUUID();
                         const sSQLInsert = this.getPendingEntityFieldINSERTSQL(newEntityFieldUUID, n);
                         try {
-                            await this.LogSQLAndExecute(ds, sSQLInsert, `SQL text to insert new entity field`);
+                            await this.LogSQLAndExecute(pool, sSQLInsert, `SQL text to insert new entity field`);
                             // if we get here, we're okay, otherwise we have an exception, which we want as it blows up transaction
                         }
                         catch (e) {
@@ -1068,7 +1097,12 @@ class ManageMetadataBase {
                         }
                     }
                 }
-            });
+                await transaction.commit();
+            }
+            catch (e) {
+                await transaction.rollback();
+                throw e;
+            }
             // if we get here now send a distinct list of the entities that had new fields to the modified entity list
             // column in the resultset is called EntityName, we dont have to dedupe them here because the method below
             // will do that for us
@@ -1087,12 +1121,12 @@ class ManageMetadataBase {
      * @param relatedEntityNameFieldMap
      * @returns
      */
-    async updateEntityFieldRelatedEntityNameFieldMap(ds, entityFieldID, relatedEntityNameFieldMap) {
+    async updateEntityFieldRelatedEntityNameFieldMap(pool, entityFieldID, relatedEntityNameFieldMap) {
         try {
             const sSQL = `EXEC [${(0, config_1.mj_core_schema)()}].spUpdateEntityFieldRelatedEntityNameFieldMap
          @EntityFieldID='${entityFieldID}',
          @RelatedEntityNameFieldMap='${relatedEntityNameFieldMap}'`;
-            await this.LogSQLAndExecute(ds, sSQL, `SQL text to update entity field related entity name field map for entity field ID ${entityFieldID}`);
+            await this.LogSQLAndExecute(pool, sSQL, `SQL text to update entity field related entity name field map for entity field ID ${entityFieldID}`);
             return true;
         }
         catch (e) {
@@ -1100,10 +1134,10 @@ class ManageMetadataBase {
             return false;
         }
     }
-    async updateExistingEntitiesFromSchema(ds, excludeSchemas) {
+    async updateExistingEntitiesFromSchema(pool, excludeSchemas) {
         try {
             const sSQL = `EXEC [${(0, config_1.mj_core_schema)()}].spUpdateExistingEntitiesFromSchema @ExcludedSchemaNames='${excludeSchemas.join(',')}'`;
-            const result = await this.LogSQLAndExecute(ds, sSQL, `SQL text to update existing entities from schema`, true);
+            const result = await this.LogSQLAndExecute(pool, sSQL, `SQL text to update existing entities from schema`, true);
             // result contains the updated entities, and there is a property of each row called Name which has the entity name that was modified
             // add these to the modified entity list if they're not already in there
             if (result && result.length > 0) {
@@ -1125,10 +1159,10 @@ class ManageMetadataBase {
         // now make sure that each of these entity names is in the modified entity list
         ManageMetadataBase._modifiedEntityList = ManageMetadataBase._modifiedEntityList.concat(newlyModifiedEntityNames);
     }
-    async updateExistingEntityFieldsFromSchema(ds, excludeSchemas) {
+    async updateExistingEntityFieldsFromSchema(pool, excludeSchemas) {
         try {
             const sSQL = `EXEC [${(0, config_1.mj_core_schema)()}].spUpdateExistingEntityFieldsFromSchema @ExcludedSchemaNames='${excludeSchemas.join(',')}'`;
-            const result = await this.LogSQLAndExecute(ds, sSQL, `SQL text to update existing entity fields from schema`, true);
+            const result = await this.LogSQLAndExecute(pool, sSQL, `SQL text to update existing entity fields from schema`, true);
             // result contains the updated entity fields
             // there is a field in there called EntityName. Get a distinct list of entity names from this and add them
             // to the modified entity list if they're not already in there
@@ -1142,10 +1176,10 @@ class ManageMetadataBase {
             return false;
         }
     }
-    async deleteUnneededEntityFields(ds, excludeSchemas) {
+    async deleteUnneededEntityFields(pool, excludeSchemas) {
         try {
             const sSQL = `EXEC [${(0, config_1.mj_core_schema)()}].spDeleteUnneededEntityFields @ExcludedSchemaNames='${excludeSchemas.join(',')}'`;
-            const result = await this.LogSQLAndExecute(ds, sSQL, `SQL text to delete unneeded entity fields`, true);
+            const result = await this.LogSQLAndExecute(pool, sSQL, `SQL text to delete unneeded entity fields`, true);
             // result contains the DELETED entity fields
             // there is a field in there called Entity. Get a distinct list of entity names from this and add them
             // to the modified entity list if they're not already in there
@@ -1159,7 +1193,7 @@ class ManageMetadataBase {
             return false;
         }
     }
-    async manageEntityFieldValuesAndValidatorFunctions(ds, excludeSchemas, currentUser, skipDBUpdate) {
+    async manageEntityFieldValuesAndValidatorFunctions(pool, excludeSchemas, currentUser, skipDBUpdate) {
         try {
             // here we want to get all of the entity fields that have check constraints attached to them. For each field that has a check constraint, we want to
             // evaluate it to see if it is a simple series of OR statements or not, if it is a simple series of OR statements, we can parse the possible values
@@ -1167,11 +1201,14 @@ class ManageMetadataBase {
             // just ignore it.
             const filter = excludeSchemas && excludeSchemas.length > 0 ? ` WHERE SchemaName NOT IN (${excludeSchemas.map(s => `'${s}'`).join(',')})` : '';
             const sSQL = `SELECT * FROM [${(0, config_1.mj_core_schema)()}].vwEntityFieldsWithCheckConstraints${filter}`;
-            const result = await ds.query(sSQL);
+            const resultResult = await pool.request().query(sSQL);
+            const result = resultResult.recordset;
             const efvSQL = `SELECT * FROM [${(0, config_1.mj_core_schema)()}].EntityFieldValue`;
-            const allEntityFieldValues = await ds.query(efvSQL);
+            const allEntityFieldValuesResult = await pool.request().query(efvSQL);
+            const allEntityFieldValues = allEntityFieldValuesResult.recordset;
             const efSQL = `SELECT * FROM [${(0, config_1.mj_core_schema)()}].vwEntityFields ORDER BY EntityID, Sequence`;
-            const allEntityFields = await ds.query(efSQL);
+            const allEntityFieldsResult = await pool.request().query(efSQL);
+            const allEntityFields = allEntityFieldsResult.recordset;
             const generationPromises = [];
             const columnLevelResults = result.filter((r) => r.EntityFieldID); // get the column level constraints
             const tableLevelResults = result.filter((r) => !r.EntityFieldID); // get the table level constraints
@@ -1186,14 +1223,15 @@ class ManageMetadataBase {
                             // 1st, flip the order of parsedValues because they come out in reverse order from SQL Server
                             parsedValues.reverse();
                             // we have parsed values from the check constraint, so sync them with the entity field values
-                            await this.syncEntityFieldValues(ds, r.EntityFieldID, parsedValues, allEntityFieldValues);
+                            await this.syncEntityFieldValues(pool, r.EntityFieldID, parsedValues, allEntityFieldValues);
                             // finally, make sure the ValueListType column within the EntityField table is set to "List" because for check constraints we only allow the values specified in the list.
                             // check to see if the ValueListType is already set to "List", if not, update it
                             const sSQLCheck = `SELECT ValueListType FROM [${(0, config_1.mj_core_schema)()}].EntityField WHERE ID='${r.EntityFieldID}'`;
-                            const checkResult = await ds.query(sSQLCheck);
+                            const checkResultResult = await pool.request().query(sSQLCheck);
+                            const checkResult = checkResultResult.recordset;
                             if (checkResult && checkResult.length > 0 && checkResult[0].ValueListType.trim().toLowerCase() !== 'list') {
                                 const sSQL = `UPDATE [${(0, config_1.mj_core_schema)()}].EntityField SET ValueListType='List' WHERE ID='${r.EntityFieldID}'`;
-                                await this.LogSQLAndExecute(ds, sSQL, `SQL text to update ValueListType for entity field ID ${r.EntityFieldID}`);
+                                await this.LogSQLAndExecute(pool, sSQL, `SQL text to update ValueListType for entity field ID ${r.EntityFieldID}`);
                             }
                         }
                         else {
@@ -1232,13 +1270,13 @@ class ManageMetadataBase {
     }
     /**
      * This method will load all generated code from the database - this is intended to be used when you are bypassing managing the metadata.
-     * @param ds
+     * @param pool
      * @param currentUser
      */
-    async loadGeneratedCode(ds, currentUser) {
+    async loadGeneratedCode(pool, currentUser) {
         try {
             // right now we're just doing validator functions which are handled here
-            return await this.manageEntityFieldValuesAndValidatorFunctions(ds, [], currentUser, true);
+            return await this.manageEntityFieldValuesAndValidatorFunctions(pool, [], currentUser, true);
         }
         catch (e) {
             (0, status_logging_1.logError)(e);
@@ -1356,7 +1394,9 @@ class ManageMetadataBase {
             // now, loop through the possible values and add any that are not already in the database
             // Step 1: for any existing value that is NOT in the list of possible Values, delete it
             let numRemoved = 0;
-            await ds.transaction(async () => {
+            const transaction = new sql.Transaction(ds);
+            await transaction.begin();
+            try {
                 for (const ev of existingValues) {
                     if (!possibleValues.find(v => v === ev.Value)) {
                         // delete the value from the database
@@ -1389,7 +1429,12 @@ class ManageMetadataBase {
                         numUpdated++;
                     }
                 }
-            });
+                await transaction.commit();
+            }
+            catch (e) {
+                await transaction.rollback();
+                throw e;
+            }
             return true;
         }
         catch (e) {
@@ -1454,19 +1499,27 @@ class ManageMetadataBase {
             (sExcludeSchemas.length > 0 ? (sExcludeTables.length > 0 ? ` AND ` : ``) + '(' + sExcludeSchemas + ')' : '');
         return sWhere;
     }
-    async createNewEntities(ds) {
+    async createNewEntities(pool) {
         try {
             const sSQL = `SELECT * FROM [${(0, config_1.mj_core_schema)()}].vwSQLTablesAndEntities WHERE EntityID IS NULL ` + this.createExcludeTablesAndSchemasFilter('');
-            const newEntities = await ds.query(sSQL);
+            const newEntitiesResult = await pool.request().query(sSQL);
+            const newEntities = newEntitiesResult.recordset;
             if (newEntities && newEntities.length > 0) {
                 const md = new core_1.Metadata();
-                await ds.transaction(async () => {
+                const transaction = new sql.Transaction(pool);
+                await transaction.begin();
+                try {
                     // wrap in a transaction so we get all of it or none of it
                     for (let i = 0; i < newEntities.length; ++i) {
                         // process each of the new entities
-                        await this.createNewEntity(ds, newEntities[i], md);
+                        await this.createNewEntity(pool, newEntities[i], md);
                     }
-                });
+                    await transaction.commit();
+                }
+                catch (e) {
+                    await transaction.rollback();
+                    throw e;
+                }
                 if (ManageMetadataBase.newEntityList.length > 0) {
                     // only do this if we actually created new entities
                     (0, core_1.LogStatus)(`   Done creating entities, refreshing metadata to reflect new entities...`);
@@ -1487,7 +1540,8 @@ class ManageMetadataBase {
         // validate all of these factors by getting the sql from SQL Server and check the result, if failure, shouldCreate=false and generate validation message, otherwise return empty validation message and true for shouldCreate.
         const query = `EXEC ${core_1.Metadata.Provider.ConfigData.MJCoreSchemaName}.spGetPrimaryKeyForTable @TableName='${newEntity.TableName}', @SchemaName='${newEntity.SchemaName}'`;
         try {
-            const result = await ds.query(query);
+            const resultResult = await ds.request().query(query);
+            const result = resultResult.recordset;
             if (result.length === 0) {
                 return { shouldCreate: false, validationMessage: "No primary key found" };
             }
@@ -1580,9 +1634,9 @@ class ManageMetadataBase {
     createNewUUID() {
         return (0, uuid_1.v4)();
     }
-    async createNewEntity(ds, newEntity, md) {
+    async createNewEntity(pool, newEntity, md) {
         try {
-            const { shouldCreate, validationMessage } = await this.shouldCreateNewEntity(ds, newEntity);
+            const { shouldCreate, validationMessage } = await this.shouldCreateNewEntity(pool, newEntity);
             if (shouldCreate) {
                 // process a single new entity
                 let newEntityName = await this.createNewEntityName(newEntity);
@@ -1596,10 +1650,10 @@ class ManageMetadataBase {
                     newEntityName = newEntityName + suffix;
                     (0, core_1.LogError)(`   >>>> WARNING: Entity name already exists, so using ${newEntityName} instead. If you did not intend for this, please rename the ${newEntity.SchemaName}.${newEntity.TableName} table in the database.`);
                 }
-                const isNewSchema = await this.isSchemaNew(ds, newEntity.SchemaName);
+                const isNewSchema = await this.isSchemaNew(pool, newEntity.SchemaName);
                 const newEntityID = this.createNewUUID();
                 const sSQLInsert = this.createNewEntityInsertSQL(newEntityID, newEntityName, newEntity, suffix);
-                await this.LogSQLAndExecute(ds, sSQLInsert, `SQL generated to create new entity ${newEntityName}`);
+                await this.LogSQLAndExecute(pool, sSQLInsert, `SQL generated to create new entity ${newEntityName}`);
                 // if we get here we created a new entity safely, otherwise we get exception
                 // add it to the new entity list
                 ManageMetadataBase.newEntityList.push(newEntityName);
@@ -1609,11 +1663,11 @@ class ManageMetadataBase {
                 if (isNewSchema && config_1.configInfo.newSchemaDefaults.CreateNewApplicationWithSchemaName) {
                     // new schema and config option is to create a new application from the schema name so do that
                     // check to see if the app already exists
-                    apps = await this.getApplicationIDForSchema(ds, newEntity.SchemaName);
+                    apps = await this.getApplicationIDForSchema(pool, newEntity.SchemaName);
                     if (!apps || apps.length === 0) {
                         // doesn't already exist, so create it
                         const appUUID = this.createNewUUID();
-                        const newAppID = await this.createNewApplication(ds, appUUID, newEntity.SchemaName, newEntity.SchemaName);
+                        const newAppID = await this.createNewApplication(pool, appUUID, newEntity.SchemaName, newEntity.SchemaName);
                         if (newAppID) {
                             apps = [newAppID];
                         }
@@ -1626,7 +1680,7 @@ class ManageMetadataBase {
                 }
                 else {
                     // not a new schema, attempt to look up the application for this schema
-                    apps = await this.getApplicationIDForSchema(ds, newEntity.SchemaName);
+                    apps = await this.getApplicationIDForSchema(pool, newEntity.SchemaName);
                 }
                 if (apps && apps.length > 0) {
                     if (config_1.configInfo.newEntityDefaults.AddToApplicationWithSchemaName) {
@@ -1635,7 +1689,7 @@ class ManageMetadataBase {
                             const sSQLInsertApplicationEntity = `INSERT INTO ${(0, config_1.mj_core_schema)()}.ApplicationEntity
                                        (ApplicationID, EntityID, Sequence) VALUES
                                        ('${appUUID}', '${newEntityID}', (SELECT ISNULL(MAX(Sequence),0)+1 FROM ${(0, config_1.mj_core_schema)()}.ApplicationEntity WHERE ApplicationID = '${appUUID}'))`;
-                            await this.LogSQLAndExecute(ds, sSQLInsertApplicationEntity, `SQL generated to add new entity ${newEntityName} to application ID: '${appUUID}'`);
+                            await this.LogSQLAndExecute(pool, sSQLInsertApplicationEntity, `SQL generated to add new entity ${newEntityName} to application ID: '${appUUID}'`);
                         }
                     }
                     else {
@@ -1656,7 +1710,7 @@ class ManageMetadataBase {
                             const sSQLInsertPermission = `INSERT INTO ${(0, config_1.mj_core_schema)()}.EntityPermission
                                                    (EntityID, RoleID, CanRead, CanCreate, CanUpdate, CanDelete) VALUES
                                                    ('${newEntityID}', '${RoleID}', ${p.CanRead ? 1 : 0}, ${p.CanCreate ? 1 : 0}, ${p.CanUpdate ? 1 : 0}, ${p.CanDelete ? 1 : 0})`;
-                            await this.LogSQLAndExecute(ds, sSQLInsertPermission, `SQL generated to add new permission for entity ${newEntityName} for role ${p.RoleName}`);
+                            await this.LogSQLAndExecute(pool, sSQLInsertPermission, `SQL generated to add new permission for entity ${newEntityName} for role ${p.RoleName}`);
                         }
                         else
                             (0, core_1.LogError)(`   >>>> ERROR: Unable to find Role ID for role ${p.RoleName} to add permissions for new entity ${newEntityName}`);
@@ -1673,16 +1727,17 @@ class ManageMetadataBase {
             (0, core_1.LogError)(`Failed to create new entity ${newEntity?.TableName}`);
         }
     }
-    async isSchemaNew(ds, schemaName) {
+    async isSchemaNew(pool, schemaName) {
         // check to see if there are any entities in the db with this schema name
         const sSQL = `SELECT COUNT(*) AS Count FROM [${(0, config_1.mj_core_schema)()}].Entity WHERE SchemaName = '${schemaName}'`;
-        const result = await ds.query(sSQL);
+        const resultResult = await pool.request().query(sSQL);
+        const result = resultResult.recordset;
         return result && result.length > 0 ? result[0].Count === 0 : true;
     }
-    async createNewApplication(ds, appID, appName, schemaName) {
+    async createNewApplication(pool, appID, appName, schemaName) {
         try {
             const sSQL = "INSERT INTO [" + (0, config_1.mj_core_schema)() + "].Application (ID, Name, Description, SchemaAutoAddNewEntities) VALUES ('" + appID + "', '" + appName + "', 'Generated for schema', '" + schemaName + "')";
-            await this.LogSQLAndExecute(ds, sSQL, `SQL generated to create new application ${appName}`);
+            await this.LogSQLAndExecute(pool, sSQL, `SQL generated to create new application ${appName}`);
             return appID; // if we get here, we successfully created the application, so return the ID
         }
         catch (e) {
@@ -1690,15 +1745,17 @@ class ManageMetadataBase {
             return null; // if we get here, we failed to create the application
         }
     }
-    async applicationExists(ds, applicationName) {
+    async applicationExists(pool, applicationName) {
         const sSQL = `SELECT ID FROM [${(0, config_1.mj_core_schema)()}].Application WHERE Name = '${applicationName}'`;
-        const result = await ds.query(sSQL);
+        const resultResult = await pool.request().query(sSQL);
+        const result = resultResult.recordset;
         return result && result.length > 0 ? result[0].ID.length > 0 : false;
     }
-    async getApplicationIDForSchema(ds, schemaName) {
+    async getApplicationIDForSchema(pool, schemaName) {
         // get all the apps each time from DB as we might be adding, don't use Metadata here for that reason
         const sSQL = `SELECT ID, Name, SchemaAutoAddNewEntities FROM [${(0, config_1.mj_core_schema)()}].vwApplications`;
-        const result = await ds.query(sSQL);
+        const resultResult = await pool.request().query(sSQL);
+        const result = resultResult.recordset;
         if (!result || result.length === 0) {
             // no applications found, return null
             return null;
@@ -1761,21 +1818,18 @@ class ManageMetadataBase {
         return sSQLInsert;
     }
     /**
-     * Executes the given SQL query using the given DataSource object.
+     * Executes the given SQL query using the given ConnectionPool object.
      * If the appendToLogFile parameter is true, the query will also be appended to the log file.
      * Note that in order to append to the log file, ManageMetadataBase.manageMetaDataLogging must be called first.
-     * @param ds - The DataSource object to use to execute the query.
+     * @param pool - The ConnectionPool object to use to execute the query.
      * @param query - The SQL query to execute.
      * @param description - A description of the query to append to the log file.
      * @param isRecurringScript - if set to true tells the logger that the provided SQL represents a recurring script meaning it is something that is executed, generally, for all CodeGen runs. In these cases, the Config settings can result in omitting these recurring scripts from being logged because the configuration environment may have those recurring scripts already set to run after all run-specific migrations get run.
      * @returns - The result of the query execution.
      */
-    async LogSQLAndExecute(ds, query, description, isRecurringScript = false) {
-        return await sql_logging_1.SQLLogging.LogSQLAndExecute(ds, query, description, isRecurringScript);
+    async LogSQLAndExecute(pool, query, description, isRecurringScript = false) {
+        return await sql_logging_1.SQLLogging.LogSQLAndExecute(pool, query, description, isRecurringScript);
     }
 }
 exports.ManageMetadataBase = ManageMetadataBase;
-ManageMetadataBase._newEntityList = [];
-ManageMetadataBase._modifiedEntityList = [];
-ManageMetadataBase._generatedValidators = [];
 //# sourceMappingURL=manage-metadata.js.map