@memberjunction/codegen-lib 5.13.0 → 5.15.0

package/dist/Database/manage-metadata.js

@@ -5,7 +5,7 @@ import { EntityInfo, ExtractActualDefaultValue, LogError, LogStatus, Metadata, S
 import { logError, logMessage, logStatus } from "../Misc/status_logging.js";
 import { SQLUtilityBase } from "./sql.js";
 import { AdvancedGeneration } from "../Misc/advanced_generation.js";
-import { SQLParser } from "@memberjunction/core-entities-server";
+import { SQLParser } from "@memberjunction/sql-parser";
 import { createDisplayName, generatePluralName, MJGlobal, stripTrailingChars, UUIDsEqual } from "@memberjunction/global";
 import { v4 as uuidv4 } from 'uuid';
 import * as fs from 'fs';
@@ -313,6 +313,189 @@ export class ManageMetadataBase {
             .filter((e) => typeof e.BaseTable === 'string' && typeof e.SchemaName === 'string')
             .map((e) => ({ ...e, BaseTable: e.BaseTable, SchemaName: e.SchemaName }));
     }
+    /**
+     * Extracts organic key configurations from the additionalSchemaInfo config file.
+     * Walks all schema-keyed table arrays and collects OrganicKeys arrays with their
+     * owning schema and table name.
+     */
+    extractOrganicKeysFromConfig(config) {
+        const results = [];
+        for (const [key, value] of Object.entries(config)) {
+            // Skip top-level non-schema keys (ISARelationships, VirtualEntities, Entities)
+            if (!Array.isArray(value))
+                continue;
+            if (['ISARelationships', 'VirtualEntities', 'Entities'].includes(key))
+                continue;
+            const schemaName = key;
+            for (const tableConfig of value) {
+                if (!tableConfig.TableName || !Array.isArray(tableConfig.OrganicKeys))
+                    continue;
+                const organicKeys = tableConfig.OrganicKeys.map((ok, okIndex) => ({
+                    Name: ok.Name,
+                    Description: ok.Description || undefined,
+                    MatchFieldNames: ok.MatchFieldNames,
+                    NormalizationStrategy: ok.NormalizationStrategy || 'LowerCaseTrim',
+                    CustomNormalizationExpression: ok.CustomNormalizationExpression || undefined,
+                    Sequence: ok.Sequence ?? okIndex,
+                    RelatedEntities: Array.isArray(ok.RelatedEntities)
+                        ? ok.RelatedEntities.map((re, reIndex) => ({
+                            SchemaName: re.SchemaName,
+                            TableName: re.TableName,
+                            RelatedFieldNames: re.RelatedFieldNames,
+                            TransitiveObject: re.TransitiveObject,
+                            TransitiveMatchFieldNames: re.TransitiveMatchFieldNames,
+                            TransitiveOutputFieldName: re.TransitiveOutputFieldName,
+                            RelatedEntityJoinFieldName: re.RelatedEntityJoinFieldName,
+                            TransitiveView: re.TransitiveView,
+                            DisplayName: re.DisplayName,
+                            DisplayLocation: re.DisplayLocation || 'After Field Tabs',
+                            Sequence: re.Sequence ?? reIndex,
+                        }))
+                        : [],
+                }));
+                results.push({ SchemaName: schemaName, TableName: tableConfig.TableName, OrganicKeys: organicKeys });
+            }
+        }
+        return results;
+    }
+    /**
+     * Processes organic key configurations from additionalSchemaInfo.
+     * For each configured organic key:
+     * 1. Creates transitive bridge views if TransitiveView is defined
+     * 2. Upserts EntityOrganicKey records (matched on EntityID + Name)
+     * 3. Upserts EntityOrganicKeyRelatedEntity records (matched on EntityOrganicKeyID + RelatedEntityID)
+     *
+     * All SQL is executed AND logged via LogSQLAndExecute for complete CI/CD traceability.
+     * Must run AFTER entities are created.
+     */
+    async processOrganicKeyConfig(pool) {
+        const config = ManageMetadataBase.getSoftPKFKConfig();
+        if (!config)
+            return { success: true, createdCount: 0, updatedCount: 0 };
+        const allOrganicKeys = this.extractOrganicKeysFromConfig(config);
+        if (allOrganicKeys.length === 0)
+            return { success: true, createdCount: 0, updatedCount: 0 };
+        const schema = mj_core_schema();
+        let createdCount = 0;
+        let updatedCount = 0;
+        for (const tableConfig of allOrganicKeys) {
+            // Resolve the owning entity
+            const ownerResult = await this.runQueryWithParams(pool, `
+            ${this.selectTop(1, 'ID, Name', `FROM ${this.qs(schema, 'vwEntities')}
+            WHERE (BaseTable = @TableName AND SchemaName = @SchemaName)
+               OR Name = @TableName`, 'CASE WHEN BaseTable = @TableName AND SchemaName = @SchemaName THEN 0 ELSE 1 END')}
+         `, { 'TableName': tableConfig.TableName, 'SchemaName': tableConfig.SchemaName });
+            if (ownerResult.recordset.length === 0) {
+                logError(`    > Organic keys config: entity "${tableConfig.SchemaName}.${tableConfig.TableName}" not found — skipping`);
+                continue;
+            }
+            const ownerEntityId = ownerResult.recordset[0].ID;
+            const ownerEntityName = ownerResult.recordset[0].Name;
+            for (const okConfig of tableConfig.OrganicKeys) {
+                try {
+                    // Step 1: Create transitive views if defined
+                    for (const re of okConfig.RelatedEntities) {
+                        if (re.TransitiveView) {
+                            const viewSchema = re.TransitiveView.SchemaName || re.SchemaName;
+                            const viewFullName = `${viewSchema}.${re.TransitiveView.Name}`;
+                            const viewSQL = `CREATE OR ALTER VIEW ${this.qs(viewSchema, re.TransitiveView.Name)} AS\n${re.TransitiveView.SQL}`;
+                            await this.LogSQLAndExecute(pool, viewSQL, `Create transitive bridge view ${viewFullName} for organic key "${okConfig.Name}" on ${ownerEntityName}`);
+                            // Auto-populate TransitiveObject from the view definition
+                            re.TransitiveObject = viewFullName;
+                        }
+                    }
+                    // Step 2: Upsert EntityOrganicKey (match on EntityID + Name)
+                    const existingKey = await this.runQueryWithParams(pool, `SELECT ID FROM ${this.qs(schema, 'EntityOrganicKey')} WHERE EntityID = @EntityID AND Name = @Name`, { 'EntityID': ownerEntityId, 'Name': okConfig.Name });
+                    let organicKeyId;
+                    const matchFieldNames = okConfig.MatchFieldNames.join(',');
+                    if (existingKey.recordset.length > 0) {
+                        // Update existing
+                        organicKeyId = existingKey.recordset[0].ID;
+                        const updateSQL = `UPDATE ${this.qs(schema, 'EntityOrganicKey')} SET
+                     MatchFieldNames = '${matchFieldNames}',
+                     NormalizationStrategy = '${okConfig.NormalizationStrategy || 'LowerCaseTrim'}',
+                     ${okConfig.CustomNormalizationExpression ? `CustomNormalizationExpression = '${okConfig.CustomNormalizationExpression.replace(/'/g, "''")}',` : ''}
+                     ${okConfig.Description ? `Description = '${okConfig.Description.replace(/'/g, "''")}',` : ''}
+                     Sequence = ${okConfig.Sequence ?? 0},
+                     Status = 'Active'
+                     WHERE ID = '${organicKeyId}'`;
+                        await this.LogSQLAndExecute(pool, updateSQL, `Update organic key "${okConfig.Name}" on ${ownerEntityName}`);
+                        updatedCount++;
+                        logStatus(`    > Organic key: Updated "${okConfig.Name}" on ${ownerEntityName}`);
+                    }
+                    else {
+                        // Insert new — generate a deterministic UUID based on entity+name for idempotency
+                        const insertSQL = `INSERT INTO ${this.qs(schema, 'EntityOrganicKey')}
+                     (EntityID, Name, ${okConfig.Description ? 'Description, ' : ''}MatchFieldNames, NormalizationStrategy, ${okConfig.CustomNormalizationExpression ? 'CustomNormalizationExpression, ' : ''}Sequence, Status)
+                     VALUES ('${ownerEntityId}', '${okConfig.Name}', ${okConfig.Description ? `'${okConfig.Description.replace(/'/g, "''")}', ` : ''}'${matchFieldNames}', '${okConfig.NormalizationStrategy || 'LowerCaseTrim'}', ${okConfig.CustomNormalizationExpression ? `'${okConfig.CustomNormalizationExpression.replace(/'/g, "''")}', ` : ''}${okConfig.Sequence ?? 0}, 'Active')`;
+                        await this.LogSQLAndExecute(pool, insertSQL, `Insert organic key "${okConfig.Name}" on ${ownerEntityName}`);
+                        createdCount++;
+                        logStatus(`    > Organic key: Created "${okConfig.Name}" on ${ownerEntityName}`);
+                        // Fetch the new ID
+                        const newKey = await this.runQueryWithParams(pool, `SELECT ID FROM ${this.qs(schema, 'EntityOrganicKey')} WHERE EntityID = @EntityID AND Name = @Name`, { 'EntityID': ownerEntityId, 'Name': okConfig.Name });
+                        organicKeyId = newKey.recordset[0].ID;
+                    }
+                    // Step 3: Upsert EntityOrganicKeyRelatedEntity for each related entity
+                    for (const reConfig of okConfig.RelatedEntities) {
+                        const relResult = await this.runQueryWithParams(pool, `
+                     ${this.selectTop(1, 'ID, Name', `FROM ${this.qs(schema, 'vwEntities')}
+                     WHERE (BaseTable = @TableName AND SchemaName = @SchemaName)
+                        OR Name = @TableName`, 'CASE WHEN BaseTable = @TableName AND SchemaName = @SchemaName THEN 0 ELSE 1 END')}
+                  `, { 'TableName': reConfig.TableName, 'SchemaName': reConfig.SchemaName });
+                        if (relResult.recordset.length === 0) {
+                            logError(`    > Organic key "${okConfig.Name}": related entity "${reConfig.SchemaName}.${reConfig.TableName}" not found — skipping`);
+                            continue;
+                        }
+                        const relEntityId = relResult.recordset[0].ID;
+                        const relEntityName = relResult.recordset[0].Name;
+                        // Check if this related entity mapping already exists
+                        const existingRel = await this.runQueryWithParams(pool, `SELECT ID FROM ${this.qs(schema, 'EntityOrganicKeyRelatedEntity')} WHERE EntityOrganicKeyID = @KeyID AND RelatedEntityID = @RelEntityID`, { 'KeyID': organicKeyId, 'RelEntityID': relEntityId });
+                        // Build field values
+                        const isDirect = reConfig.RelatedFieldNames && reConfig.RelatedFieldNames.length > 0;
+                        const relFieldNames = isDirect ? reConfig.RelatedFieldNames.join(',') : null;
+                        const transitiveObject = reConfig.TransitiveObject || null;
+                        const transitiveMatchFields = reConfig.TransitiveMatchFieldNames ? reConfig.TransitiveMatchFieldNames.join(',') : null;
+                        if (existingRel.recordset.length > 0) {
+                            const relId = existingRel.recordset[0].ID;
+                            const updateRelSQL = `UPDATE ${this.qs(schema, 'EntityOrganicKeyRelatedEntity')} SET
+                        RelatedEntityFieldNames = ${relFieldNames ? `'${relFieldNames}'` : 'NULL'},
+                        TransitiveObjectName = ${transitiveObject ? `'${transitiveObject}'` : 'NULL'},
+                        TransitiveObjectMatchFieldNames = ${transitiveMatchFields ? `'${transitiveMatchFields}'` : 'NULL'},
+                        TransitiveObjectOutputFieldName = ${reConfig.TransitiveOutputFieldName ? `'${reConfig.TransitiveOutputFieldName}'` : 'NULL'},
+                        RelatedEntityJoinFieldName = ${reConfig.RelatedEntityJoinFieldName ? `'${reConfig.RelatedEntityJoinFieldName}'` : 'NULL'},
+                        DisplayName = ${reConfig.DisplayName ? `'${reConfig.DisplayName.replace(/'/g, "''")}'` : 'NULL'},
+                        DisplayLocation = '${reConfig.DisplayLocation || 'After Field Tabs'}',
+                        Sequence = ${reConfig.Sequence ?? 0}
+                        WHERE ID = '${relId}'`;
+                            await this.LogSQLAndExecute(pool, updateRelSQL, `Update organic key related entity: "${okConfig.Name}" → ${relEntityName}`);
+                        }
+                        else {
+                            const insertRelSQL = `INSERT INTO ${this.qs(schema, 'EntityOrganicKeyRelatedEntity')}
+                        (EntityOrganicKeyID, RelatedEntityID, RelatedEntityFieldNames,
+                         TransitiveObjectName, TransitiveObjectMatchFieldNames, TransitiveObjectOutputFieldName, RelatedEntityJoinFieldName,
+                         DisplayName, DisplayLocation, Sequence)
+                        VALUES ('${organicKeyId}', '${relEntityId}',
+                         ${relFieldNames ? `'${relFieldNames}'` : 'NULL'},
+                         ${transitiveObject ? `'${transitiveObject}'` : 'NULL'},
+                         ${transitiveMatchFields ? `'${transitiveMatchFields}'` : 'NULL'},
+                         ${reConfig.TransitiveOutputFieldName ? `'${reConfig.TransitiveOutputFieldName}'` : 'NULL'},
+                         ${reConfig.RelatedEntityJoinFieldName ? `'${reConfig.RelatedEntityJoinFieldName}'` : 'NULL'},
+                         ${reConfig.DisplayName ? `'${reConfig.DisplayName.replace(/'/g, "''")}'` : 'NULL'},
+                         '${reConfig.DisplayLocation || 'After Field Tabs'}',
+                         ${reConfig.Sequence ?? 0})`;
+                            await this.LogSQLAndExecute(pool, insertRelSQL, `Insert organic key related entity: "${okConfig.Name}" → ${relEntityName}`);
+                        }
+                        logStatus(`    > Organic key "${okConfig.Name}": ${existingRel.recordset.length > 0 ? 'updated' : 'created'} → ${relEntityName} (${isDirect ? 'direct' : 'transitive'})`);
+                    }
+                }
+                catch (err) {
+                    const errMessage = err instanceof Error ? err.message : String(err);
+                    logError(`    > Organic key config: Failed to process "${okConfig.Name}" on ${ownerEntityName}: ${errMessage}`);
+                }
+            }
+        }
+        return { success: true, createdCount, updatedCount };
+    }
     /**
      * Processes IS-A relationship configurations from the additionalSchemaInfo config.
      * For each configured relationship, looks up both entities by name (or by table name
@@ -626,6 +809,12 @@ export class ManageMetadataBase {
         if (entityConfigResult.updatedCount > 0) {
             logStatus(`    > Updated attributes on ${entityConfigResult.updatedCount} entit${entityConfigResult.updatedCount === 1 ? 'y' : 'ies'} from config`);
         }
+        // Config-driven organic key setup — create bridge views and upsert organic key metadata
+        // Must run AFTER entities exist
+        const organicKeyResult = await this.processOrganicKeyConfig(pool);
+        if (organicKeyResult.createdCount > 0 || organicKeyResult.updatedCount > 0) {
+            logStatus(`    > Organic keys: ${organicKeyResult.createdCount} created, ${organicKeyResult.updatedCount} updated from config`);
+        }
         start = new Date();
         logStatus('   Syncing schema info from database...');
         if (!await this.updateSchemaInfoFromDatabase(pool, excludeSchemas)) {
@@ -880,11 +1069,11 @@ export class ManageMetadataBase {
      * for the LLM to use when decorating virtual entity fields.
      */
     buildSourceEntityContext(viewDefinition) {
-        const parseResult = SQLParser.Parse(viewDefinition);
+        const tableRefs = SQLParser.ExtractTableRefs(viewDefinition);
         const md = new Metadata();
         const sourceEntities = [];
         const seen = new Set();
-        for (const tableRef of parseResult.Tables) {
+        for (const tableRef of tableRefs) {
             // Match against MJ entities by BaseTable/BaseView + SchemaName
             const matchingEntity = md.Entities.find(e => (e.BaseTable.toLowerCase() === tableRef.TableName.toLowerCase() ||
                 e.BaseView.toLowerCase() === tableRef.TableName.toLowerCase()) &&