@memberjunction/codegen-lib 6.1.0-edge.4 → 6.1.0-edge.6

package/dist/Database/sql_codegen.js

@@ -1,4 +1,4 @@
-import { EntityInfo, Metadata } from '@memberjunction/core';
+import { EntityInfo, Metadata, ShouldJoinRecordGeoCodes, HasNativeLatLngFields, NativeLatitudeField, NativeLongitudeField, ListEmbeddedGeoSpecs } from '@memberjunction/core';
 import { logError, logStatus, logWarning, startSpinner, updateSpinner, succeedSpinner, failSpinner } from '../Misc/status_logging.js';
 import * as fs from 'fs';
 import path from 'path';
@@ -12,6 +12,7 @@ import { combineFiles, logIf } from '../Misc/util.js';
 import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
 import { SQLLogging } from '../Misc/sql_logging.js';
 import { TempBatchFile } from '../Misc/temp_batch_file.js';
+import { writeFileIfChanged as writeFileIfChangedShared } from '../Misc/file-write.js';
 export const SPType = {
     Create: 'Create',
     Update: 'Update',
@@ -67,6 +68,13 @@ export class SQLCodeGenBase {
          * Ordered list of entity IDs for delete SP regeneration (dependency order)
          */
         this.orderedEntitiesForDeleteSPRegeneration = [];
+        /**
+         * `schema.routine` keys (lowercased) present in the database at the start of
+         * this SQL-generation pass. Used to force-log CREATE PROC for routines that
+         * were dropped out-of-band (BigSchemaDemo recreate, a failed prior batch)
+         * while the entity itself is not in newEntityList/modifiedEntityList.
+         */
+        this.existingRoutines = null;
     }
     get SQLUtilityObject() {
         return this._sqlUtilityObject;
@@ -79,6 +87,7 @@ export class SQLCodeGenBase {
     }
     async manageSQLScriptsAndExecution(pool, entities, directory, currentUser) {
         try {
+            this.existingRoutines = null;
             // Build list of entities qualified for forced regeneration if entityWhereClause is provided
             if (configInfo.forceRegeneration?.enabled && configInfo.forceRegeneration?.entityWhereClause) {
                 this.filterEntitiesQualifiedForRegeneration = true; // Enable filtering
@@ -114,6 +123,7 @@ export class SQLCodeGenBase {
                 return false;
             }
             succeedSpinner(`Custom SQL scripts completed (${(new Date().getTime() - startTime.getTime()) / 1000}s)`);
+            await this.loadExistingRoutines(pool, entities);
             // ALWAYS use the first filter where we only include entities that have IncludeInAPI = 1
             // Entities are already sorted by name in PostProcessEntityMetadata (see providerBase.ts)
             const baselineEntities = entities.filter(e => e.IncludeInAPI);
@@ -187,7 +197,8 @@ export class SQLCodeGenBase {
             // AI Engine init because vwAIModels is gone. SQL Server doesn't
             // hit this because its execution path is bulk-monolithic, not
             // phased per-entity.
-            const perEntityBatchSize = configInfo.dbPlatform === 'postgresql' ? 1 : 5;
+            const configuredBatch = configInfo.fileEmit?.sqlEntityBatchSize ?? 8;
+            const perEntityBatchSize = configInfo.dbPlatform === 'postgresql' ? 1 : configuredBatch;
             // Generate SQL for entities that don't need cascade delete regeneration
             const genResult = await this.generateAndExecuteEntitySQLToSeparateFiles({
                 pool,
@@ -669,6 +680,8 @@ export class SQLCodeGenBase {
         // BaseView while CodeGen keeps writing the inner view underneath it. Gating on
         // BaseViewGenerated alone would skip the inner view and leave the custom layer selecting
         // from an object that does not exist.
+        // One pass: overlay already applied → Pass 1 sees extra BaseView columns and logs
+        // EntityField INSERTs; Pass 2 DROPs/creates only GeneratedViewName, never the overlay.
         const generatesView = (entity.BaseViewGenerated || entity.HasLayeredBaseView) && !entity.VirtualEntity;
         const tvfSQL = generatesView ? this.generateRecursiveFKTVFs(entity) : '';
         const viewPieces = generatesView
@@ -719,14 +732,7 @@ export class SQLCodeGenBase {
      * This avoids false timestamp updates and unnecessary I/O.
      */
     writeFileIfChanged(filePath, newContent) {
-        if (fs.existsSync(filePath)) {
-            const existing = fs.readFileSync(filePath, 'utf-8');
-            if (existing === newContent) {
-                return false;
-            }
-        }
-        fs.writeFileSync(filePath, newContent);
-        return true;
+        return writeFileIfChangedShared(filePath, newContent);
     }
     /**
      * Fetches the current base view definition from SQL Server and compares it against newly generated SQL.
@@ -1106,7 +1112,7 @@ export class SQLCodeGenBase {
                     if (options.writeFiles) {
                         const filePath = path.join(options.directory, this.SQLUtilityObject.getDBObjectFileName('sp', options.entity.SchemaName, spName, false, true));
                         this.writeFileIfChanged(filePath, s);
-                        this.logSQLForNewOrModifiedEntity(options.entity, s, `spCreate SQL for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, baseViewChanged);
+                        this.logSQLForNewOrModifiedEntity(options.entity, s, `spCreate SQL for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, this.forceLogForRoutine(options.entity, spName, baseViewChanged));
                         files.push(filePath);
                     }
                     sRet += s + '\n' + this._dbProvider.BatchSeparator + '\n';
@@ -1117,7 +1123,7 @@ export class SQLCodeGenBase {
                 if (options.writeFiles) {
                     const filePath = path.join(options.directory, this.SQLUtilityObject.getDBObjectFileName('sp', options.entity.SchemaName, spName, true, true));
                     this.writeFileIfChanged(filePath, s);
-                    this.logSQLForNewOrModifiedEntity(options.entity, s, `spCreate Permissions for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, baseViewChanged);
+                    this.logSQLForNewOrModifiedEntity(options.entity, s, `spCreate Permissions for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, this.forceLogForRoutine(options.entity, spName, baseViewChanged));
                     files.push(filePath);
                 }
                 // now, append the permissions to the return string IF we did NOT generate the proc - because if we generated the proc, that
@@ -1136,7 +1142,7 @@ export class SQLCodeGenBase {
                     if (options.writeFiles) {
                         const filePath = path.join(options.directory, this.SQLUtilityObject.getDBObjectFileName('sp', options.entity.SchemaName, spName, false, true));
                         this.writeFileIfChanged(filePath, s);
-                        this.logSQLForNewOrModifiedEntity(options.entity, s, `spUpdate SQL for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, baseViewChanged);
+                        this.logSQLForNewOrModifiedEntity(options.entity, s, `spUpdate SQL for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, this.forceLogForRoutine(options.entity, spName, baseViewChanged));
                         files.push(filePath);
                     }
                     sRet += s + '\n' + this._dbProvider.BatchSeparator + '\n';
@@ -1147,7 +1153,7 @@ export class SQLCodeGenBase {
                 if (options.writeFiles) {
                     const filePath = path.join(options.directory, this.SQLUtilityObject.getDBObjectFileName('sp', options.entity.SchemaName, spName, true, true));
                     this.writeFileIfChanged(filePath, s);
-                    this.logSQLForNewOrModifiedEntity(options.entity, s, `spUpdate Permissions for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, baseViewChanged);
+                    this.logSQLForNewOrModifiedEntity(options.entity, s, `spUpdate Permissions for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, this.forceLogForRoutine(options.entity, spName, baseViewChanged));
                     files.push(filePath);
                 }
                 // now, append the permissions to the return string IF we did NOT generate the proc - because if we generated the proc, that
@@ -1172,7 +1178,7 @@ export class SQLCodeGenBase {
                     if (options.writeFiles) {
                         const filePath = path.join(options.directory, this.SQLUtilityObject.getDBObjectFileName('sp', options.entity.SchemaName, spName, false, true));
                         this.writeFileIfChanged(filePath, s);
-                        this.logSQLForNewOrModifiedEntity(options.entity, s, `spDelete SQL for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, baseViewChanged);
+                        this.logSQLForNewOrModifiedEntity(options.entity, s, `spDelete SQL for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, this.forceLogForRoutine(options.entity, spName, baseViewChanged));
                         files.push(filePath);
                     }
                     sRet += s + '\n' + this._dbProvider.BatchSeparator + '\n';
@@ -1183,7 +1189,7 @@ export class SQLCodeGenBase {
                 if (options.writeFiles) {
                     const filePath = path.join(options.directory, this.SQLUtilityObject.getDBObjectFileName('sp', options.entity.SchemaName, spName, true, true));
                     this.writeFileIfChanged(filePath, s);
-                    this.logSQLForNewOrModifiedEntity(options.entity, s, `spDelete Permissions for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, baseViewChanged);
+                    this.logSQLForNewOrModifiedEntity(options.entity, s, `spDelete Permissions for ${options.entity.Name}`, options.enableSQLLoggingForNewOrModifiedEntities, this.forceLogForRoutine(options.entity, spName, baseViewChanged));
                     files.push(filePath);
                 }
                 // now, append the permissions to the return string IF we did NOT generate the proc - because if we generated the proc, that
@@ -1229,6 +1235,42 @@ export class SQLCodeGenBase {
     getSPName(entity, type) {
         return this._dbProvider.getCRUDRoutineName(entity, type);
     }
+    /**
+     * Snapshot every CRUD routine currently in the database (one query) so
+     * generate-and-log can force-emit CREATE PROC for objects that vanished
+     * without the entity landing on newEntityList/modifiedEntityList.
+     */
+    async loadExistingRoutines(pool, entities) {
+        this.existingRoutines = new Set();
+        const schemas = [...new Set(entities.map((e) => e.SchemaName).filter((s) => !!s))];
+        const sql = this._dbProvider.getRoutineNamesBySchemaSQL(schemas);
+        if (!sql || !sql.trim()) {
+            return;
+        }
+        try {
+            const result = await pool.query(sql);
+            for (const row of result.recordset ?? []) {
+                const schemaName = String(row.schema_name ?? '').toLowerCase();
+                const routineName = String(row.routine_name ?? '').toLowerCase();
+                if (schemaName && routineName) {
+                    this.existingRoutines.add(`${schemaName}.${routineName}`);
+                }
+            }
+        }
+        catch (e) {
+            logWarning(`Could not load existing CRUD routines (missing-proc self-heal disabled this run): ${e instanceof Error ? e.message : String(e)}`);
+            this.existingRoutines = null;
+        }
+    }
+    isRoutineMissing(schema, routineName) {
+        if (!this.existingRoutines) {
+            return false;
+        }
+        return !this.existingRoutines.has(`${schema}.${routineName}`.toLowerCase());
+    }
+    forceLogForRoutine(entity, routineName, baseViewChanged) {
+        return baseViewChanged || this.isRoutineMissing(entity.SchemaName, routineName);
+    }
     getEntityPermissionFileNames(entity) {
         const files = [];
         // all entities have a base view - and we always generate permissions for the base view even if not generated base view
@@ -1556,17 +1598,22 @@ export class SQLCodeGenBase {
         const classNameFirstChar = entity.BaseTableCodeName.charAt(0).toLowerCase();
         let relatedFieldsString = await this.generateBaseViewRelatedFieldsString(pool, entity.Fields);
         const relatedFieldsJoinString = this.generateBaseViewJoins(entity, entity.Fields);
-        // Geo support: add __mj_Latitude and __mj_Longitude virtual fields for geo-enabled entities
-        // Skip for Record Geo Codes itself to avoid circular self-reference in the view
+        // Geo WRITE source: native lat/lng aliases, or RecordGeoCode JOIN when the entity
+        // has writable Geo* fields and no native coords. Display-only entities (Person/Org
+        // virtual PrimaryAddress*) do not get a RecordGeoCode join on their own ID.
         if (entity.SupportsGeoCoding && entity.Name.trim().toLowerCase() !== 'mj: record geo codes') {
             const qi = this._dbProvider.Dialect.QuoteIdentifier.bind(this._dbProvider.Dialect);
-            const geoFieldsSelect = this.hasNativeGeoFields(entity.Fields)
+            const geoFieldsSelect = HasNativeLatLngFields(entity.Fields)
                 ? this.generateNativeGeoFields(entity.Fields, classNameFirstChar, qi)
-                : this.generateRecordGeoCodeFields(qi);
+                : (ShouldJoinRecordGeoCodes(entity) ? this.generateRecordGeoCodeFields(qi) : '');
             if (geoFieldsSelect) {
                 relatedFieldsString += (relatedFieldsString ? ',\n' : '') + geoFieldsSelect;
             }
         }
+        const embeddedGeo = this.generateEmbeddedGeoSelect(entity, classNameFirstChar);
+        if (embeddedGeo.select) {
+            relatedFieldsString += (relatedFieldsString ? ',\n' : '') + embeddedGeo.select;
+        }
         // GRANTs target the PUBLIC view (BaseView), not the one this method generates. For a
         // LAYERED entity those are different objects, and the outer one may not exist yet: the
         // documented setup is "name the inner view -> run CodeGen -> then create BaseView", so the
@@ -1680,10 +1727,9 @@ export class SQLCodeGenBase {
                 sOutput += `${ef.AllowsNull ? 'LEFT OUTER' : 'INNER'} JOIN\n    ${qs(ef.RelatedEntitySchemaName, relatedTable)} AS ${ef._RelatedEntityTableAlias}\n  ON\n    ${qi(classNameFirstChar)}.${qi(ef.Name)} = ${ef._RelatedEntityTableAlias}.${qi(ef.RelatedEntityFieldName)}`;
             }
         }
-        // Geo support: add LEFT JOIN to vwRecordGeoCodes for entities with SupportsGeoCoding = 1
-        // that don't have native GeoLatitude/GeoLongitude fields (those get aliased directly).
-        // Skip for Record Geo Codes itself to avoid circular self-reference in the view.
-        if (entity.SupportsGeoCoding && !this.hasNativeGeoFields(entityFields) && entity.Name.trim().toLowerCase() !== 'mj: record geo codes') {
+        // Geo WRITE source: RecordGeoCode JOIN only when there are writable Geo* fields
+        // and no native lat/lng. Display-only entities skip this join.
+        if (ShouldJoinRecordGeoCodes(entity)) {
             const dialect = this._dbProvider.Dialect;
             const qi = dialect.QuoteIdentifier.bind(dialect);
             const qs = dialect.QuoteSchema.bind(dialect);
@@ -1707,6 +1753,7 @@ export class SQLCodeGenBase {
             }
             sOutput += `LEFT OUTER JOIN\n    ${qs(mjCoreSchema, 'vwRecordGeoCodes')} AS __mj_rgc\n  ON\n    __mj_rgc.${qi('EntityID')} = '${entity.ID}'\n    AND __mj_rgc.${qi('RecordID')} = ${recordIdExpr}\n    AND __mj_rgc.${qi('LocationType')} = 'Primary'`;
         }
+        sOutput += this.generateEmbeddedGeoJoins(entity, classNameFirstChar);
         return sOutput;
     }
     /**
@@ -1723,21 +1770,53 @@ export class SQLCodeGenBase {
      * on any geo-eligible entity whose RecordGeoCode-based view ran once.
      */
     hasNativeGeoFields(entityFields) {
-        const hasLat = entityFields.some(f => f.ExtendedType === 'GeoLatitude' && !f.IsVirtual);
-        const hasLng = entityFields.some(f => f.ExtendedType === 'GeoLongitude' && !f.IsVirtual);
-        return hasLat && hasLng;
+        return HasNativeLatLngFields(entityFields);
     }
     /**
      * Generate SELECT aliases for native lat/lng fields → __mj_Latitude / __mj_Longitude.
      * Virtual fields are excluded — see hasNativeGeoFields for the rationale.
      */
     generateNativeGeoFields(entityFields, classNameFirstChar, qi) {
-        const latField = entityFields.find(f => f.ExtendedType === 'GeoLatitude' && !f.IsVirtual);
-        const lngField = entityFields.find(f => f.ExtendedType === 'GeoLongitude' && !f.IsVirtual);
+        const latField = NativeLatitudeField(entityFields);
+        const lngField = NativeLongitudeField(entityFields);
         if (!latField || !lngField)
             return '';
         return `    ${qi(classNameFirstChar)}.${qi(latField.Name)} AS ${qi('__mj_Latitude')},\n    ${qi(classNameFirstChar)}.${qi(lngField.Name)} AS ${qi('__mj_Longitude')}`;
     }
+    /**
+     * Bubble lat/lng from EmbeddedRecord peers as `__mj_Latitude_{FK}`.
+     * Geocode lives on the Address (or other source); the parent only displays it.
+     */
+    generateEmbeddedGeoSelect(entity, classNameFirstChar) {
+        const specs = ListEmbeddedGeoSpecs(entity.Fields);
+        if (specs.length === 0)
+            return { select: '', joins: '' };
+        const md = new Metadata(); // global-provider-ok: CodeGen is CLI tool
+        const qi = this._dbProvider.Dialect.QuoteIdentifier.bind(this._dbProvider.Dialect);
+        const qs = this._dbProvider.Dialect.QuoteSchema.bind(this._dbProvider.Dialect);
+        const selects = [];
+        const joins = [];
+        for (const spec of specs) {
+            const related = md.Entities.find(e => e.ID.toLowerCase() === spec.relatedEntityID.toLowerCase());
+            if (!related)
+                continue;
+            const latF = NativeLatitudeField(related.Fields) ?? related.Fields.find(f => f.Name === '__mj_Latitude' || f.Name === 'PrimaryAddressLatitude');
+            const lngF = NativeLongitudeField(related.Fields) ?? related.Fields.find(f => f.Name === '__mj_Longitude' || f.Name === 'PrimaryAddressLongitude');
+            if (!latF || !lngF)
+                continue;
+            const alias = `__mj_emb_${spec.foreignKeyField}`;
+            const schema = spec.relatedSchemaName || related.SchemaName;
+            const table = spec.relatedBaseTable || related.BaseTable;
+            const joinKind = spec.allowsNull ? 'LEFT OUTER' : 'INNER';
+            joins.push(`${joinKind} JOIN\n    ${qs(schema, table)} AS ${alias}\n  ON\n    ${qi(classNameFirstChar)}.${qi(spec.foreignKeyField)} = ${alias}.${qi(related.FirstPrimaryKey.Name)}`); // first-pk-ok: FK target — a single FK column joins to the related entity's single key
+            selects.push(`    ${alias}.${qi(latF.Name)} AS ${qi(spec.lat)},\n    ${alias}.${qi(lngF.Name)} AS ${qi(spec.lng)}`);
+        }
+        return { select: selects.join(',\n'), joins: joins.join('\n') };
+    }
+    generateEmbeddedGeoJoins(entity, classNameFirstChar) {
+        const { joins } = this.generateEmbeddedGeoSelect(entity, classNameFirstChar);
+        return joins ? ((entity.Fields.length ? '\n' : '') + joins) : '';
+    }
     /**
      * Generate SELECT aliases from the RecordGeoCode LEFT JOIN → __mj_Latitude / __mj_Longitude.
      */