@memberjunction/codegen-lib 2.103.0 → 2.105.0

package/dist/Database/sql_codegen.js

@@ -38,6 +38,7 @@ const sqlserver_dataprovider_1 = require("@memberjunction/sqlserver-dataprovider
 const util_1 = require("../Misc/util");
 const global_1 = require("@memberjunction/global");
 const sql_logging_1 = require("../Misc/sql_logging");
+const temp_batch_file_1 = require("../Misc/temp_batch_file");
 exports.SPType = {
     Create: 'Create',
     Update: 'Update',
@@ -110,9 +111,15 @@ class SQLCodeGenBase {
             }
             (0, status_logging_1.succeedSpinner)(`Custom SQL scripts completed (${(new Date().getTime() - startTime.getTime()) / 1000}s)`);
             // ALWAYS use the first filter where we only include entities that have IncludeInAPI = 1
-            const baselineEntities = entities.filter(e => e.IncludeInAPI);
+            // Sort entities by name for deterministic processing order (workaround until MJCore fix in issue #1436)
+            const sortedEntities = entities.sort((a, b) => a.Name.localeCompare(b.Name));
+            const baselineEntities = sortedEntities.filter(e => e.IncludeInAPI);
             const includedEntities = baselineEntities.filter(e => config_1.configInfo.excludeSchemas.find(s => s.toLowerCase() === e.SchemaName.toLowerCase()) === undefined); //only include entities that are NOT in the excludeSchemas list
             const excludedEntities = baselineEntities.filter(e => config_1.configInfo.excludeSchemas.find(s => s.toLowerCase() === e.SchemaName.toLowerCase()) !== undefined); //only include entities that ARE in the excludeSchemas list in this array
+            // Initialize temp batch files for each schema
+            // These will be populated as SQL is generated and will be used for actual execution
+            const schemas = Array.from(new Set(baselineEntities.map(e => e.SchemaName)));
+            temp_batch_file_1.TempBatchFile.initialize(directory, schemas);
             // STEP 1.5 - Check for cascade delete dependencies that require regeneration
             (0, status_logging_1.startSpinner)('Analyzing cascade delete dependencies...');
             await this.markEntitiesForCascadeDeleteRegeneration(pool, includedEntities);
@@ -184,11 +191,27 @@ class SQLCodeGenBase {
             (0, status_logging_1.startSpinner)('Creating combined SQL files...');
             const allEntityFiles = this.createCombinedEntitySQLFiles(directory, baselineEntities);
             (0, status_logging_1.succeedSpinner)(`Created combined SQL files for ${allEntityFiles.length} schemas`);
-            // STEP 2(e) ---- FINALLY, we now execute all the combined files by schema;
-            (0, status_logging_1.startSpinner)('Executing combined entity SQL files...');
+            // STEP 2(e) ---- FINALLY, we execute SQL in proper dependency order
+            // Use temp batch files (which maintain CodeGen log order) if available, otherwise fall back to combined files
+            (0, status_logging_1.startSpinner)('Executing entity SQL files...');
             const step2eStartTime = new Date();
-            if (!await this.SQLUtilityObject.executeSQLFiles(allEntityFiles, config_1.configInfo?.verboseOutput ?? false)) {
-                (0, status_logging_1.failSpinner)('Failed to execute combined entity SQL files');
+            let executionSuccess = false;
+            if (temp_batch_file_1.TempBatchFile.hasContent()) {
+                // Execute temp batch files in dependency order (matches CodeGen run log)
+                const tempFiles = temp_batch_file_1.TempBatchFile.getTempFilePaths();
+                (0, util_1.logIf)(config_1.configInfo?.verboseOutput ?? false, `Executing ${tempFiles.length} temp batch file(s) in dependency order`);
+                executionSuccess = await this.SQLUtilityObject.executeSQLFiles(tempFiles, config_1.configInfo?.verboseOutput ?? false);
+                // Clean up temp files after execution
+                temp_batch_file_1.TempBatchFile.cleanup();
+            }
+            else {
+                // Fall back to combined files (for backward compatibility or if temp files weren't created)
+                (0, util_1.logIf)(config_1.configInfo?.verboseOutput ?? false, `Executing ${allEntityFiles.length} combined file(s)`);
+                executionSuccess = await this.SQLUtilityObject.executeSQLFiles(allEntityFiles, config_1.configInfo?.verboseOutput ?? false);
+            }
+            if (!executionSuccess) {
+                (0, status_logging_1.failSpinner)('Failed to execute entity SQL files');
+                temp_batch_file_1.TempBatchFile.cleanup(); // Cleanup on error
                 return false;
             }
             const step2eEndTime = new Date();
@@ -227,6 +250,8 @@ class SQLCodeGenBase {
         }
         catch (err) {
             (0, status_logging_1.logError)(err);
+            // Clean up temp batch files on error
+            temp_batch_file_1.TempBatchFile.cleanup();
             return false;
         }
     }
@@ -446,6 +471,8 @@ class SQLCodeGenBase {
         }
         if (shouldLog) {
             sql_logging_1.SQLLogging.appendToSQLLogFile(sql, description);
+            // Also write to temp batch file for actual execution (matches CodeGen log order)
+            temp_batch_file_1.TempBatchFile.appendToTempBatchFile(sql, entity.SchemaName);
         }
         (0, util_1.logIf)(config_1.configInfo.verboseOutput, `SQL Generated for ${entity.Name}: ${description}`);
     }
@@ -475,8 +502,10 @@ class SQLCodeGenBase {
                 sRet += s + '\nGO\n';
             }
             // BASE VIEW
+            // Only generate if BaseViewGenerated is true (respects custom views where it's false)
+            // forceRegeneration.baseViews only forces regeneration of views where BaseViewGenerated=true
             if (!options.onlyPermissions &&
-                (options.entity.BaseViewGenerated || (config_1.configInfo.forceRegeneration?.enabled && config_1.configInfo.forceRegeneration?.baseViews)) &&
+                options.entity.BaseViewGenerated &&
                 !options.entity.VirtualEntity) {
                 // generate the base view
                 const s = this.generateSingleEntitySQLFileHeader(options.entity, options.entity.BaseView) + await this.generateBaseView(options.pool, options.entity);
@@ -505,9 +534,9 @@ class SQLCodeGenBase {
             // CREATE SP
             if (options.entity.AllowCreateAPI && !options.entity.VirtualEntity) {
                 const spName = this.getSPName(options.entity, exports.SPType.Create);
-                if (!options.onlyPermissions &&
-                    (options.entity.spCreateGenerated ||
-                        (config_1.configInfo.forceRegeneration?.enabled && (config_1.configInfo.forceRegeneration?.spCreate || config_1.configInfo.forceRegeneration?.allStoredProcedures)))) {
+                // Only generate if spCreateGenerated is true (respects custom SPs where it's false)
+                // forceRegeneration only forces regeneration of SPs where spCreateGenerated=true
+                if (!options.onlyPermissions && options.entity.spCreateGenerated) {
                     // generate the create SP
                     const s = this.generateSingleEntitySQLFileHeader(options.entity, spName) + this.generateSPCreate(options.entity);
                     if (options.writeFiles) {
@@ -535,9 +564,9 @@ class SQLCodeGenBase {
             // UPDATE SP
             if (options.entity.AllowUpdateAPI && !options.entity.VirtualEntity) {
                 const spName = this.getSPName(options.entity, exports.SPType.Update);
-                if (!options.onlyPermissions &&
-                    (options.entity.spUpdateGenerated ||
-                        (config_1.configInfo.forceRegeneration?.enabled && (config_1.configInfo.forceRegeneration?.spUpdate || config_1.configInfo.forceRegeneration?.allStoredProcedures)))) {
+                // Only generate if spUpdateGenerated is true (respects custom SPs where it's false)
+                // forceRegeneration only forces regeneration of SPs where spUpdateGenerated=true
+                if (!options.onlyPermissions && options.entity.spUpdateGenerated) {
                     // generate the update SP
                     const s = this.generateSingleEntitySQLFileHeader(options.entity, spName) + this.generateSPUpdate(options.entity);
                     if (options.writeFiles) {
@@ -565,9 +594,11 @@ class SQLCodeGenBase {
             // DELETE SP
             if (options.entity.AllowDeleteAPI && !options.entity.VirtualEntity) {
                 const spName = this.getSPName(options.entity, exports.SPType.Delete);
+                // Only generate if spDeleteGenerated is true (respects custom SPs where it's false)
+                // OR if this entity has cascade delete dependencies that require regeneration
+                // forceRegeneration only forces regeneration of SPs where spDeleteGenerated=true
                 if (!options.onlyPermissions &&
-                    (options.entity.spDeleteGenerated || // Generate if marked as generated (not custom)
-                        (config_1.configInfo.forceRegeneration?.enabled && (config_1.configInfo.forceRegeneration?.spDelete || config_1.configInfo.forceRegeneration?.allStoredProcedures)) ||
+                    (options.entity.spDeleteGenerated ||
                         this.entitiesNeedingDeleteSPRegeneration.has(options.entity.ID))) {
                     // generate the delete SP
                     if (this.entitiesNeedingDeleteSPRegeneration.has(options.entity.ID)) {
@@ -634,11 +665,11 @@ class SQLCodeGenBase {
     getSPName(entity, type) {
         switch (type) {
             case exports.SPType.Create:
-                return entity.spCreate && entity.spCreate.length > 0 ? entity.spCreate : 'spCreate' + entity.ClassName;
+                return entity.spCreate && entity.spCreate.length > 0 ? entity.spCreate : 'spCreate' + entity.BaseTableCodeName;
             case exports.SPType.Update:
-                return entity.spUpdate && entity.spUpdate.length > 0 ? entity.spUpdate : 'spUpdate' + entity.ClassName;
+                return entity.spUpdate && entity.spUpdate.length > 0 ? entity.spUpdate : 'spUpdate' + entity.BaseTableCodeName;
             case exports.SPType.Delete:
-                return entity.spDelete && entity.spDelete.length > 0 ? entity.spDelete : 'spDelete' + entity.ClassName;
+                return entity.spDelete && entity.spDelete.length > 0 ? entity.spDelete : 'spDelete' + entity.BaseTableCodeName;
         }
     }
     getEntityPermissionFileNames(entity) {
@@ -865,6 +896,77 @@ CREATE INDEX ${indexName} ON [${entity.SchemaName}].[${entity.BaseTable}] ([${f.
         }
         return sOutput;
     }
+    /**
+     * Detects self-referential foreign keys in an entity (e.g., ParentTaskID pointing back to Task table)
+     * Returns array of field info objects representing recursive relationships
+     */
+    detectRecursiveForeignKeys(entity) {
+        return entity.Fields.filter(field => field.RelatedEntityID != null &&
+            field.RelatedEntityID === entity.ID);
+    }
+    /**
+     * Generates the WITH clause containing recursive CTEs for root ID calculation
+     * Each recursive FK gets its own CTE that traverses the hierarchy to find the root
+     */
+    generateRecursiveCTEs(entity, recursiveFKs) {
+        const primaryKey = entity.FirstPrimaryKey.Name;
+        const schemaName = entity.SchemaName;
+        const tableName = entity.BaseTable;
+        const classNameFirstChar = entity.ClassName.charAt(0).toLowerCase();
+        const ctes = recursiveFKs.map(field => {
+            const fieldName = field.Name;
+            const rootFieldName = `Root${fieldName}`;
+            const cteName = `CTE_${rootFieldName}`;
+            return `    ${cteName} AS (
+        -- Anchor: rows with no parent (root nodes)
+        SELECT
+            [${primaryKey}],
+            [${primaryKey}] AS [${rootFieldName}]
+        FROM
+            [${schemaName}].[${tableName}]
+        WHERE
+            [${fieldName}] IS NULL
+
+        UNION ALL
+
+        -- Recursive: traverse up the hierarchy
+        SELECT
+            child.[${primaryKey}],
+            parent.[${rootFieldName}]
+        FROM
+            [${schemaName}].[${tableName}] child
+        INNER JOIN
+            ${cteName} parent ON child.[${fieldName}] = parent.[${primaryKey}]
+    )`;
+        }).join(',\n');
+        return `WITH\n${ctes}\n`;
+    }
+    /**
+     * Generates the SELECT clause additions for root fields
+     * Example: , cte_root.[RootParentTaskID]
+     */
+    generateRootFieldSelects(recursiveFKs, classNameFirstChar) {
+        return recursiveFKs.map(field => {
+            const rootFieldName = `Root${field.Name}`;
+            const cteName = `CTE_${rootFieldName}`;
+            return `,\n    ${cteName}.[${rootFieldName}]`;
+        }).join('');
+    }
+    /**
+     * Generates LEFT OUTER JOINs to the recursive CTEs
+     */
+    generateRecursiveCTEJoins(recursiveFKs, classNameFirstChar, entity) {
+        if (recursiveFKs.length === 0) {
+            return '';
+        }
+        const primaryKey = entity.FirstPrimaryKey.Name;
+        const joins = recursiveFKs.map(field => {
+            const rootFieldName = `Root${field.Name}`;
+            const cteName = `CTE_${rootFieldName}`;
+            return `LEFT OUTER JOIN\n    ${cteName}\n  ON\n    [${classNameFirstChar}].[${primaryKey}] = ${cteName}.[${primaryKey}]`;
+        }).join('\n');
+        return '\n' + joins;
+    }
     async generateBaseView(pool, entity) {
         const viewName = entity.BaseView ? entity.BaseView : `vw${entity.CodeName}`;
         const classNameFirstChar = entity.ClassName.charAt(0).toLowerCase();
@@ -874,6 +976,10 @@ CREATE INDEX ${indexName} ON [${entity.SchemaName}].[${entity.BaseTable}] ([${f.
         const whereClause = entity.DeleteType === 'Soft' ? `WHERE
     ${classNameFirstChar}.[${core_1.EntityInfo.DeletedAtFieldName}] IS NULL
 ` : '';
+        // Detect recursive foreign keys and generate CTEs
+        const recursiveFKs = this.detectRecursiveForeignKeys(entity);
+        const cteClause = recursiveFKs.length > 0 ? this.generateRecursiveCTEs(entity, recursiveFKs) : '';
+        const rootFields = recursiveFKs.length > 0 ? this.generateRootFieldSelects(recursiveFKs, classNameFirstChar) : '';
         return `
 ------------------------------------------------------------
 ----- BASE VIEW FOR ENTITY:      ${entity.Name}
@@ -881,15 +987,16 @@ CREATE INDEX ${indexName} ON [${entity.SchemaName}].[${entity.BaseTable}] ([${f.
 -----               BASE TABLE:  ${entity.BaseTable}
 -----               PRIMARY KEY: ${entity.PrimaryKeys.map(pk => pk.Name).join(', ')}
 ------------------------------------------------------------
-DROP VIEW IF EXISTS [${entity.SchemaName}].[${viewName}]
+IF OBJECT_ID('[${entity.SchemaName}].[${viewName}]', 'V') IS NOT NULL
+    DROP VIEW [${entity.SchemaName}].[${viewName}];
 GO
 
 CREATE VIEW [${entity.SchemaName}].[${viewName}]
 AS
-SELECT
-    ${classNameFirstChar}.*${relatedFieldsString.length > 0 ? ',' : ''}${relatedFieldsString}
+${cteClause}SELECT
+    ${classNameFirstChar}.*${relatedFieldsString.length > 0 ? ',' : ''}${relatedFieldsString}${rootFields}
 FROM
-    [${entity.SchemaName}].[${entity.BaseTable}] AS ${classNameFirstChar}${relatedFieldsJoinString ? '\n' + relatedFieldsJoinString : ''}
+    [${entity.SchemaName}].[${entity.BaseTable}] AS ${classNameFirstChar}${relatedFieldsJoinString ? '\n' + relatedFieldsJoinString : ''}${this.generateRecursiveCTEJoins(recursiveFKs, classNameFirstChar, entity)}
 ${whereClause}GO${permissions}
     `;
     }
@@ -999,7 +1106,7 @@ ${whereClause}GO${permissions}
         return (sOutput == '' ? '' : '\n') + sOutput;
     }
     generateSPCreate(entity) {
-        const spName = entity.spCreate ? entity.spCreate : `spCreate${entity.ClassName}`;
+        const spName = entity.spCreate ? entity.spCreate : `spCreate${entity.BaseTableCodeName}`;
         const firstKey = entity.FirstPrimaryKey;
         //double exclamations used on the firstKey.DefaultValue property otherwise the type of this variable is 'number | ""';
         const primaryKeyAutomatic = firstKey.AutoIncrement; // Only exclude auto-increment fields, allow manual override for all other PKs including UUIDs with defaults
@@ -1081,7 +1188,8 @@ ${whereClause}GO${permissions}
 ------------------------------------------------------------
 ----- CREATE PROCEDURE FOR ${entity.BaseTable}
 ------------------------------------------------------------
-DROP PROCEDURE IF EXISTS [${entity.SchemaName}].[${spName}]
+IF OBJECT_ID('[${entity.SchemaName}].[${spName}]', 'P') IS NOT NULL
+    DROP PROCEDURE [${entity.SchemaName}].[${spName}];
 GO
 
 CREATE PROCEDURE [${entity.SchemaName}].[${spName}]
@@ -1113,7 +1221,8 @@ GO${permissions}
 ------------------------------------------------------------
 ----- TRIGGER FOR ${core_1.EntityInfo.UpdatedAtFieldName} field for the ${entity.BaseTable} table
 ------------------------------------------------------------
-DROP TRIGGER IF EXISTS [${entity.SchemaName}].trgUpdate${entity.ClassName}
+IF OBJECT_ID('[${entity.SchemaName}].[trgUpdate${entity.ClassName}]', 'TR') IS NOT NULL
+    DROP TRIGGER [${entity.SchemaName}].[trgUpdate${entity.ClassName}];
 GO
 CREATE TRIGGER [${entity.SchemaName}].trgUpdate${entity.ClassName}
 ON [${entity.SchemaName}].[${entity.BaseTable}]
@@ -1135,7 +1244,7 @@ GO`;
         return triggerStatement;
     }
     generateSPUpdate(entity) {
-        const spName = entity.spUpdate ? entity.spUpdate : `spUpdate${entity.ClassName}`;
+        const spName = entity.spUpdate ? entity.spUpdate : `spUpdate${entity.BaseTableCodeName}`;
         const efParamString = this.createEntityFieldsParamString(entity.Fields, true);
         const permissions = this.generateSPPermissions(entity, spName, exports.SPType.Update);
         const hasUpdatedAtField = entity.Fields.find(f => f.Name.toLowerCase().trim() === core_1.EntityInfo.UpdatedAtFieldName.trim().toLowerCase()) !== undefined;
@@ -1151,7 +1260,8 @@ GO`;
 ------------------------------------------------------------
 ----- UPDATE PROCEDURE FOR ${entity.BaseTable}
 ------------------------------------------------------------
-DROP PROCEDURE IF EXISTS [${entity.SchemaName}].[${spName}]
+IF OBJECT_ID('[${entity.SchemaName}].[${spName}]', 'P') IS NOT NULL
+    DROP PROCEDURE [${entity.SchemaName}].[${spName}];
 GO
 
 CREATE PROCEDURE [${entity.SchemaName}].[${spName}]
@@ -1277,7 +1387,7 @@ ${updatedAtTrigger}
         return sOutput;
     }
     generateSPDelete(entity) {
-        const spName = entity.spDelete ? entity.spDelete : `spDelete${entity.ClassName}`;
+        const spName = entity.spDelete ? entity.spDelete : `spDelete${entity.BaseTableCodeName}`;
         const sCascadeDeletes = this.generateCascadeDeletes(entity);
         const permissions = this.generateSPPermissions(entity, spName, exports.SPType.Delete);
         let sVariables = '';
@@ -1318,7 +1428,8 @@ ${deleteCode}        AND ${core_1.EntityInfo.DeletedAtFieldName} IS NULL -- don'
 ------------------------------------------------------------
 ----- DELETE PROCEDURE FOR ${entity.BaseTable}
 ------------------------------------------------------------
-DROP PROCEDURE IF EXISTS [${entity.SchemaName}].[${spName}]
+IF OBJECT_ID('[${entity.SchemaName}].[${spName}]', 'P') IS NOT NULL
+    DROP PROCEDURE [${entity.SchemaName}].[${spName}];
 GO
 
 CREATE PROCEDURE [${entity.SchemaName}].[${spName}]
@@ -1370,7 +1481,7 @@ GO${permissions}
         else if (fkField.AllowsNull && !relatedEntity.AllowUpdateAPI) {
             // Nullable FK but no update API - this is a configuration error
             const sqlComment = `WARNING: ${relatedEntity.BaseTable} has nullable FK to ${parentEntity.BaseTable} but doesn't allow update API - cascade operation will fail`;
-            const consoleMsg = `WARNING in spDelete${parentEntity.ClassName} generation: ${relatedEntity.BaseTable} has nullable FK to ${parentEntity.BaseTable} but doesn't allow update API - cascade operation will fail`;
+            const consoleMsg = `WARNING in spDelete${parentEntity.BaseTableCodeName} generation: ${relatedEntity.BaseTable} has nullable FK to ${parentEntity.BaseTable} but doesn't allow update API - cascade operation will fail`;
             (0, status_logging_1.logWarning)(consoleMsg);
             return `
     -- ${sqlComment}
@@ -1379,7 +1490,7 @@ GO${permissions}
         else if (!relatedEntity.AllowDeleteAPI) {
             // Entity doesn't allow delete API, so we can't cascade delete
             const sqlComment = `WARNING: ${relatedEntity.BaseTable} has non-nullable FK to ${parentEntity.BaseTable} but doesn't allow delete API - cascade operation will fail`;
-            const consoleMsg = `WARNING in spDelete${parentEntity.ClassName} generation: ${relatedEntity.BaseTable} has non-nullable FK to ${parentEntity.BaseTable} but doesn't allow delete API - cascade operation will fail`;
+            const consoleMsg = `WARNING in spDelete${parentEntity.BaseTableCodeName} generation: ${relatedEntity.BaseTable} has non-nullable FK to ${parentEntity.BaseTable} but doesn't allow delete API - cascade operation will fail`;
             (0, status_logging_1.logWarning)(consoleMsg);
             return `
     -- ${sqlComment}
@@ -1519,6 +1630,11 @@ GO${permissions}
             for (const e of md.Entities) {
                 for (const ef of e.Fields) {
                     if (ef.RelatedEntityID === entity.ID && ef.IsVirtual === false) {
+                        // Skip self-referential foreign keys (e.g., ParentID pointing to same entity)
+                        // These don't create inter-entity dependencies for ordering purposes
+                        if (e.ID === entity.ID) {
+                            continue;
+                        }
                         // Check if this would generate a cascade operation
                         const wouldGenerateOperation = (ef.AllowsNull === false && e.AllowDeleteAPI) || // Non-nullable FK: cascade delete
                             (ef.AllowsNull && e.AllowUpdateAPI); // Nullable FK: cascade update
@@ -1699,23 +1815,27 @@ GO${permissions}
                 }
             }
         }
-        // Topological sort using DFS
+        // Topological sort using DFS with circular dependency handling
+        const circularDeps = new Set();
         const visit = (entityId) => {
             if (visited.has(entityId)) {
                 return true;
             }
             if (visiting.has(entityId)) {
-                // Circular dependency detected
+                // Circular dependency detected - mark it but don't fail
                 const entity = entities.find(e => e.ID === entityId);
                 (0, status_logging_1.logStatus)(`Warning: Circular cascade delete dependency detected involving ${entity?.Name || entityId}`);
-                return false;
+                circularDeps.add(entityId);
+                return false; // Signal circular dependency but continue processing
             }
             visiting.add(entityId);
             // Visit dependencies first
             const dependencies = reverseMap.get(entityId) || new Set();
             for (const depId of dependencies) {
                 if (!visit(depId)) {
-                    return false;
+                    // If dependency visit failed (circular), skip this dependency edge
+                    // but continue processing other dependencies
+                    continue;
                 }
             }
             visiting.delete(entityId);
@@ -1726,9 +1846,19 @@ GO${permissions}
         // Visit all entities that need ordering
         for (const entityId of entityIdsToOrder) {
             if (!visited.has(entityId)) {
-                visit(entityId);
+                const success = visit(entityId);
+                if (!success && circularDeps.has(entityId)) {
+                    // Entity is part of circular dependency - add it anyway in arbitrary order
+                    // The SQL will still be generated, just not in perfect dependency order
+                    (0, status_logging_1.logStatus)(`  - Adding ${entities.find(e => e.ID === entityId)?.Name || entityId} despite circular dependency`);
+                    visited.add(entityId);
+                    ordered.push(entityId);
+                }
             }
         }
+        if (circularDeps.size > 0) {
+            (0, status_logging_1.logStatus)(`Note: ${circularDeps.size} entities have circular cascade delete dependencies and will be regenerated in arbitrary order.`);
+        }
         return ordered;
     }
 }