@memberjunction/codegen-lib 6.1.0-edge.3 → 6.1.0-edge.4

package/dist/Database/sql_codegen.js

@@ -4,7 +4,8 @@ import * as fs from 'fs';
 import path from 'path';
 import { SQLUtilityBase } from './sql.js';
 import { CodeGenDatabaseProvider } from './codeGenDatabaseProvider.js';
-import { autoIndexForeignKeys, configInfo, customSqlScripts, mjCoreSchema } from '../Config/config.js';
+import { autoIndexForeignKeys, autoIndexSoftPrimaryKeys, configInfo, customSqlScripts, mjCoreSchema } from '../Config/config.js';
+import { entityInCustomBaseViewRefreshScope, shouldEmitCascadeForRelatedEntity } from './schema-filters.js';
 import { ManageMetadataBase } from './manage-metadata.js';
 import { UserCache } from '@memberjunction/generic-database-provider';
 import { combineFiles, logIf } from '../Misc/util.js';
@@ -16,6 +17,15 @@ export const SPType = {
     Update: 'Update',
     Delete: 'Delete',
 };
+/**
+ * Entities whose SchemaName is not in excludeSchemas (case-insensitive).
+ * This is the CodeGen run's in-scope set: SQL objects and GRANTs are
+ * generated and executed only for these. excludeSchemas means "not this run".
+ */
+export function entitiesNotInExcludedSchemas(entities, excludeSchemas) {
+    const exclude = new Set(excludeSchemas.map((s) => s.toLowerCase()));
+    return entities.filter((e) => !exclude.has(e.SchemaName.toLowerCase()));
+}
 /**
  * This class is responsible for generating database level objects like views, stored procedures, permissions, etc. You can sub-class this class to create your own SQL generation logic or implement support for other
  * databases. The base class implements support for SQL Server. In future versions of MJ, we will break out an abstract base class that has the skeleton of the logic and then the SQL Server version will be a sub-class
@@ -108,9 +118,7 @@ export class SQLCodeGenBase {
             // Entities are already sorted by name in PostProcessEntityMetadata (see providerBase.ts)
             const baselineEntities = entities.filter(e => e.IncludeInAPI);
             // OPTIMIZATION: Use a Set for O(1) lookups instead of O(n) array finds to improve performance
-            const excludeSchemasSet = new Set(configInfo.excludeSchemas.map(s => s.toLowerCase()));
-            const includedEntities = baselineEntities.filter(e => !excludeSchemasSet.has(e.SchemaName.toLowerCase())); //only include entities that are NOT in the excludeSchemas list
-            const excludedEntities = baselineEntities.filter(e => excludeSchemasSet.has(e.SchemaName.toLowerCase())); //only include entities that ARE in the excludeSchemas list in this array
+            const includedEntities = entitiesNotInExcludedSchemas(baselineEntities, configInfo.excludeSchemas);
             // Initialize temp batch files for each schema
             // These will be populated as SQL is generated and will be used for actual execution
             const schemas = Array.from(new Set(baselineEntities.map(e => e.SchemaName))).sort();
@@ -140,17 +148,17 @@ export class SQLCodeGenBase {
                 .filter(e => e !== undefined);
             // Track per-batch success without short-circuiting. Historically,
             // any single batch failure here returned `false` immediately, which
-            // skipped the cascade-regen and excluded-perms batches and left
-            // their entities with no SQL emitted at all. The post-run CRUD
-            // validator (in runCodeGen.ts) is the authoritative ship-gate now,
-            // so we want every batch to attempt its work and surface the
-            // complete picture of what's missing — partial generation is
-            // strictly more useful than nothing.
+            // skipped the cascade-regen batch and left those entities with no
+            // SQL emitted at all. The post-run CRUD validator (in runCodeGen.ts)
+            // is the authoritative ship-gate now, so we want every in-scope
+            // batch to attempt its work and surface the complete picture of
+            // what's missing — partial generation is strictly more useful than
+            // nothing.
             let entityGenSuccess = true;
             // Build a UNION will-regenerate set across every batch we're about
-            // to launch (main + cascade-regen + excluded-perms). The PG view
-            // fallback's restoreDependents path uses this to skip restoring
-            // captured dependents that codegen will recreate later in the run.
+            // to launch (main + cascade-regen). The PG view fallback's
+            // restoreDependents path uses this to skip restoring captured
+            // dependents that codegen will recreate later in the run.
             // Without the union, batch 1's fallback only sees batch 1's
             // entities — so a dependent like vwAIAgentRuns (which lives in
             // batch 2 cascade-regen) is "missing from willRegenerate", the
@@ -162,7 +170,6 @@ export class SQLCodeGenBase {
             const allBatchEntities = [
                 ...entitiesWithoutCascadeRegeneration,
                 ...entitiesForCascadeRegeneration,
-                ...excludedEntities,
             ];
             const globalWillRegenerate = new Set(allBatchEntities
                 .filter(e => !!e.BaseView)
@@ -196,7 +203,7 @@ export class SQLCodeGenBase {
                 willRegenerate: globalWillRegenerate
             }); // enable sql logging for NEW entities....
             if (!genResult.Success) {
-                logError('Main entity SQL generation batch had failures — continuing with cascade-regen and excluded-perms batches; validator will report any missing routines.');
+                logError('Main entity SQL generation batch had failures — continuing with cascade-regen; validator will report any missing routines.');
                 entityGenSuccess = false;
             }
             // Generate SQL for cascade delete regenerations in dependency order (sequentially)
@@ -219,23 +226,19 @@ export class SQLCodeGenBase {
                 }
                 genResult.Files.push(...cascadeGenResult.Files);
             }
-            // STEP 2(c) - for the excludedEntities, while we don't want to generate SQL, we do want to generate the permissions files for them
-            updateSpinner(`Generating permissions for ${excludedEntities.length} excluded entities...`);
-            const genResult2 = await this.generateAndExecuteEntitySQLToSeparateFiles({
-                pool,
-                entities: excludedEntities,
-                directory,
-                onlyPermissions: true,
-                skipExecution: true, // skip execution because we execute it all in a giant batch below
-                batchSize: perEntityBatchSize,
-                writeFiles: true,
-                enableSQLLoggingForNewOrModifiedEntities: false, /*don't log this stuff, it is just permissions for excluded entities*/
-                willRegenerate: globalWillRegenerate
-            });
-            if (!genResult2.Success) {
-                logError('Excluded-entities permissions batch had failures — continuing; validator will report any missing routines.');
-                entityGenSuccess = false;
-            }
+            // STEP 2(c) used to generate+apply GRANT files for excludeSchemas entities
+            // ("we don't want to generate SQL, we do want to generate the permissions").
+            // That is wrong once excludeSchemas means "not this CodeGen run":
+            //
+            //   - Open Apps exclude sibling schemas (__mj_BizAppsCommon, …) and __mj.
+            //     GRANT SELECT ON [__mj_BizAppsCommon].[vwContactMethods] then runs from
+            //     the orders (or other) repo, pollutes SQL Scripts/generated/<sibling>/,
+            //     and fails the whole run (exit 1) when that object is missing or lives
+            //     in a schema the executing login does not resolve.
+            //   - Core (__mj) permissions already ship in MJ migrations. Refreshing them
+            //     from an app CodeGen is not this app's job.
+            //
+            // Permissions are applied in STEP 4 for includedEntities only.
             if (entityGenSuccess) {
                 succeedSpinner(`Entity generation completed (${(new Date().getTime() - step2StartTime.getTime()) / 1000}s)`);
             }
@@ -244,15 +247,15 @@ export class SQLCodeGenBase {
             }
             // STEP 2(d) now that we've generated the SQL, let's create a combined file in each schema sub-directory for convenience for a DBA
             startSpinner('Creating combined SQL files...');
-            const allEntityFiles = this.createCombinedEntitySQLFiles(directory, baselineEntities);
-            succeedSpinner(`Created combined SQL files for ${allEntityFiles.length} schemas`);
+            const allEntityFiles = this.createCombinedEntitySQLFiles(directory, includedEntities);
+            succeedSpinner(`Created combined SQL files for ${allEntityFiles.length} file(s)`);
             // STEP 2(e) ---- FINALLY, we execute SQL in proper dependency order
             // Use temp batch files (which maintain CodeGen log order) if available, otherwise fall back to combined files
             startSpinner('Executing entity SQL files...');
             const step2eStartTime = new Date();
             let executionSuccess = false;
             if (useProviderPhasedExecution) {
-                // Per-entity phased execution already ran during steps 2(b)/(c).
+                // Per-entity phased execution already ran during step 2(b).
                 // Skip the bulk file re-execution — replaying the same SQL here
                 // would re-trigger 42P16 without the phased recovery wrapper and
                 // undo the work the phased executor just did.
@@ -363,10 +366,13 @@ export class SQLCodeGenBase {
                     succeedSpinner('No valid entities to regenerate');
                 }
             }
-            // STEP 4- Apply permissions, executing all .permissions files
+            // STEP 4- Apply permissions for THIS run's schemas only. Excluded-schema
+            // GRANT files are not generated (see STEP 2(c) note) and must not be
+            // executed: a missing sibling object would fail CodeGen with
+            // "Cannot find the object 'vw…'" even though this app's work succeeded.
             startSpinner('Applying permissions...');
             const step4StartTime = new Date();
-            if (!await this.applyPermissions(pool, directory, baselineEntities)) {
+            if (!await this.applyPermissions(pool, directory, includedEntities)) {
                 failSpinner('Failed to apply permissions');
                 overallSuccess = false;
             }
@@ -925,7 +931,8 @@ export class SQLCodeGenBase {
         return md.Entities.filter(e => modifiedOrNewNames.includes(e.Name) &&
             !e.BaseViewGenerated &&
             e.IncludeInAPI &&
-            !e.VirtualEntity);
+            !e.VirtualEntity &&
+            entityInCustomBaseViewRefreshScope(e.SchemaName, configInfo.excludeSchemas ?? [], configInfo.includeSchemas));
     }
     /**
      * Builds a combined SQL string of sp_refreshview statements for the given entities.
@@ -951,7 +958,14 @@ export class SQLCodeGenBase {
                 return outer; // fully custom — one view, and it is a standing prerequisite
             }
             const inner = this._dbProvider.generateViewRefreshSQL(e.SchemaName, e.GeneratedViewName);
-            return inner + '\n' + this.guardOnApplicationOwnedView(e, outer);
+            const rebind = this._dbProvider.generateLayeredOuterRebindSQL(e);
+            const outerSql = outer.trim().length > 0
+                ? this.guardOnApplicationOwnedView(e, outer)
+                : '';
+            const rebindSql = rebind.trim().length > 0
+                ? this.guardOnApplicationOwnedView(e, rebind.trim())
+                : '';
+            return [inner, outerSql, rebindSql].filter((s) => s && s.trim().length > 0).join('\n');
         })
             .join('\n');
     }
@@ -981,9 +995,21 @@ export class SQLCodeGenBase {
             let baseViewChanged = false;
             // Indexes for Fkeys for the table (skip for virtual entities — views can't have indexes)
             if (!options.onlyPermissions && !options.entity.VirtualEntity) {
-                const shouldGenerateIndexes = autoIndexForeignKeys() || (configInfo.forceRegeneration?.enabled && configInfo.forceRegeneration?.indexes);
+                const forceIndexes = !!(configInfo.forceRegeneration?.enabled && configInfo.forceRegeneration?.indexes);
+                const shouldGenerateIndexes = autoIndexForeignKeys() || forceIndexes;
                 const indexSQL = shouldGenerateIndexes ? this.generateIndexesForForeignKeys(options.pool, options.entity) : ''; // generate indexes if auto-indexing is on OR force regeneration is enabled
-                const s = this.generateSingleEntitySQLFileHeader(options.entity, 'Index for Foreign Keys') + indexSQL;
+                // The soft-PK index rides in the same generated file (and therefore the same
+                // migration log), but on its own setting — see generateIndexForSoftPrimaryKey.
+                // Appended rather than prepended so the file's existing content stays
+                // byte-identical for every entity that has no soft PK, which is nearly all of
+                // them: writeFileIfChanged compares content, and reordering would rewrite the
+                // whole tree for no functional gain.
+                const softPKSQL = (autoIndexSoftPrimaryKeys() || forceIndexes)
+                    ? this.generateIndexForSoftPrimaryKey(options.entity)
+                    : '';
+                const s = this.generateSingleEntitySQLFileHeader(options.entity, 'Index for Foreign Keys')
+                    + indexSQL
+                    + (softPKSQL ? (indexSQL ? '\n\n' : '') + softPKSQL : '');
                 if (options.writeFiles) {
                     const filePath = path.join(options.directory, this.SQLUtilityObject.getDBObjectFileName('index', options.entity.SchemaName, options.entity.BaseTable, false, true));
                     this.writeFileIfChanged(filePath, s);
@@ -1324,6 +1350,19 @@ export class SQLCodeGenBase {
         const indexStatements = this._dbProvider.generateForeignKeyIndexes(entity);
         return indexStatements.join('\n\n');
     }
+    /**
+     * Generates the composite index covering the entity's SOFT primary key. Empty for every
+     * entity with a real `PRIMARY KEY` constraint, which is nearly all of them.
+     *
+     * Kept separate from {@link generateIndexesForForeignKeys} and gated by its own setting: a
+     * soft-PK index serves MJ's own per-record existence check on the create path, which is a
+     * different concern from indexing foreign keys for joins and filters, and an operator's
+     * opinion about one is not an opinion about the other.
+     */
+    generateIndexForSoftPrimaryKey(entity) {
+        const indexStatements = this._dbProvider.generateSoftPrimaryKeyIndex(entity);
+        return indexStatements.join('\n\n');
+    }
     /**
      * Detects self-referential foreign keys in an entity (e.g., ParentTaskID pointing back to Task table)
      * Returns array of field info objects representing recursive relationships
@@ -1597,6 +1636,12 @@ export class SQLCodeGenBase {
             const refreshSQL = this._dbProvider.generateViewRefreshSQL(entity.SchemaName, entity.BaseView);
             sOutput += this.guardOnApplicationOwnedView(entity, refreshSQL) + separator;
         }
+        if (entity.HasLayeredBaseView && !entity.VirtualEntity) {
+            const rebindSQL = this._dbProvider.generateLayeredOuterRebindSQL(entity);
+            if (rebindSQL.trim().length > 0) {
+                sOutput += this.guardOnApplicationOwnedView(entity, rebindSQL.trim()) + separator;
+            }
+        }
         return sOutput + permissionsHeader + this.guardOnApplicationOwnedView(entity, permissionsBody) + separator;
     }
     /**
@@ -1949,6 +1994,9 @@ export class SQLCodeGenBase {
             const md = new Metadata(); // global-provider-ok: codegen runs offline against a single provider
             // Find all fields in other entities that are foreign keys to this entity
             for (const e of md.Entities) {
+                if (!shouldEmitCascadeForRelatedEntity(entity.SchemaName, e.SchemaName, configInfo.allowCrossSchemaCascadeDeletes === true)) {
+                    continue;
+                }
                 for (const ef of e.Fields) {
                     if (UUIDsEqual(ef.RelatedEntityID, entity.ID) && ef.IsVirtual === false) {
                         const cascadeSql = await this.generateSingleCascadeOperation(entity, e, ef, pool);
@@ -2059,6 +2107,9 @@ export class SQLCodeGenBase {
             const md = new Metadata(); // global-provider-ok: codegen runs offline against a single provider
             // Find all fields in other entities that are foreign keys to this entity
             for (const e of md.Entities) {
+                if (!shouldEmitCascadeForRelatedEntity(entity.SchemaName, e.SchemaName, configInfo.allowCrossSchemaCascadeDeletes === true)) {
+                    continue;
+                }
                 for (const ef of e.Fields) {
                     if (UUIDsEqual(ef.RelatedEntityID, entity.ID) && ef.IsVirtual === false) {
                         // Skip self-referential foreign keys (e.g., ParentID pointing to same entity)