@memberjunction/codegen-lib 5.44.0 → 5.45.1

package/dist/Database/manage-metadata.js

@@ -5,12 +5,13 @@ import { CodeGenDatabaseProvider } from './codeGenDatabaseProvider.js';
 // and the SS code path silently fails.
 import './providers/sqlserver/SQLServerCodeGenProvider.js';
 import { configInfo, currentWorkingDirectory, dbPlatform, getSettingValue, mj_core_schema, outputDir } from '../Config/config.js';
-import { EntityInfo, ExtractActualDefaultValue, LogError, LogStatus, Metadata, SeverityType } from "@memberjunction/core";
+import { EntityInfo, ExternalDataSourceReadRouter, ExtractActualDefaultValue, LogError, LogStatus, Metadata, SeverityType } from "@memberjunction/core";
 import { MJEntityFieldSchema } from "@memberjunction/core-entities";
 import { logError, logMessage, logStatus, startSpinner, updateSpinner, succeedSpinner } from "../Misc/status_logging.js";
 import { SQLUtilityBase } from "./sql.js";
 import { AdvancedGeneration } from "../Misc/advanced_generation.js";
 import { CodeGenReporter } from "../Misc/codegen-reporter.js";
+import { mapExternalNativeTypeToMJ } from "../Misc/externalTypeMapping.js";
 import { SQLParser } from "@memberjunction/sql-parser";
 import { createDisplayName, generatePluralName, MJGlobal, stripTrailingChars, UUIDsEqual } from "@memberjunction/global";
 import { v4 as uuidv4 } from 'uuid';
@@ -44,6 +45,23 @@ export class ManageMetadataBase {
         this._dbProvider = null;
         // ─── End Dialect Infrastructure ───────────────────────────────────
         this._sqlUtilityObject = MJGlobal.Instance.ClassFactory.CreateInstance(SQLUtilityBase);
+        /**
+         * External-data-source analogue of {@link manageVirtualEntities}. For each entity backed by an
+         * external data source, introspect the REMOTE schema (via the EDS router resolved through the
+         * ClassFactory) and sync its `EntityField` rows to match — the remote equivalent of reading the
+         * local INFORMATION_SCHEMA for a virtual/view entity. No-op when there are no external entities.
+         */
+        /**
+         * Whether the current database's `Entity` table has the `ExternalDataSourceID` column. External
+         * Data Sources ship as a SQL Server migration only, so on any database/schema that predates it
+         * (PostgreSQL today) the column is absent — and ANY raw SQL referencing it throws
+         * "column does not exist" and aborts the entire CodeGen run, even for a pure MJ-DB schema with no
+         * external entities. Callers gate EDS-specific queries on this so CodeGen stays green everywhere
+         * and auto-activates once the column lands on other platforms (mirrors the PG stored-proc guard in
+         * metadataSupportObjects.ts). Cross-platform via INFORMATION_SCHEMA; cached for the run (the schema
+         * cannot change mid-run).
+         */
+        this._entityHasExternalDataSourceColumn = null;
     }
     /**
      * Returns the CodeGenDatabaseProvider for the current database platform.
@@ -962,6 +980,23 @@ export class ManageMetadataBase {
             logError('   Error managing virtual entities');
             bSuccess = false;
         }
+        // External-data-source entities: introspect the REMOTE schema and sync their EntityField rows
+        // (the remote analogue of reading the local view's columns for a virtual entity).
+        const eeResult = await this.manageExternalEntities(pool, currentUser);
+        if (!eeResult.success) {
+            logError('   Error managing external data source entities');
+            bSuccess = false;
+        }
+        // External entities are processed AFTER the main manageEntityRelationships pass above, so any
+        // foreign keys they introspected (now set as soft FKs) need a second relationship pass to be
+        // materialized into EntityRelationship records in this same run.
+        if (eeResult.relationshipsUpdated) {
+            logStatus('   Managing external-entity relationships...');
+            if (!await this.manageEntityRelationships(pool, excludeSchemas, md)) {
+                logError('   Error managing external entity relationships');
+                bSuccess = false;
+            }
+        }
         // LLM-assisted virtual entity field decoration — identify PKs, FKs, and descriptions
         await this.decorateVirtualEntitiesWithLLM(pool, currentUser);
         // Config-driven IS-A relationship setup — set ParentID on child entities
@@ -1079,6 +1114,257 @@ export class ManageMetadataBase {
         }
         return { success: bSuccess, anyUpdates: anyUpdates };
     }
+    async entityHasExternalDataSourceColumn(pool) {
+        if (this._entityHasExternalDataSourceColumn === null) {
+            // Check the VIEW the gated queries actually read (vwEntities), not the base Entity table — a
+            // schema where the table has the column but the view wasn't refreshed to expose it would still
+            // throw. (Matches the PG guard in metadataSupportObjects.ts, which also checks the view.)
+            const sql = `SELECT COUNT(*) AS ColExists FROM INFORMATION_SCHEMA.COLUMNS ` +
+                `WHERE TABLE_SCHEMA = '${mj_core_schema()}' AND TABLE_NAME = 'vwEntities' AND COLUMN_NAME = 'ExternalDataSourceID'`;
+            const result = await this.runQuery(pool, sql);
+            const row = (result.recordset?.[0] ?? {});
+            const cnt = row.ColExists ?? row.colexists ?? 0;
+            this._entityHasExternalDataSourceColumn = Number(cnt) > 0;
+        }
+        return this._entityHasExternalDataSourceColumn;
+    }
+    async manageExternalEntities(pool, currentUser) {
+        let bSuccess = true;
+        let anyUpdates = false;
+        let relationshipsUpdated = false;
+        // EDS is provisioned only where its (SQL Server) migration ran. On a database whose Entity table
+        // lacks ExternalDataSourceID (e.g. PostgreSQL), there can be no external entities — and the query
+        // below would throw and abort the whole CodeGen run — so skip cleanly.
+        if (!(await this.entityHasExternalDataSourceColumn(pool))) {
+            return { success: true, anyUpdates: false, relationshipsUpdated: false };
+        }
+        const sql = `SELECT * FROM ${this.qs(mj_core_schema(), 'vwEntities')} WHERE ExternalDataSourceID IS NOT NULL`;
+        const result = await this.runQuery(pool, sql);
+        const externalEntities = result.recordset;
+        if (!externalEntities || externalEntities.length === 0) {
+            return { success: true, anyUpdates: false, relationshipsUpdated: false };
+        }
+        // Lazily load the External Data Sources engine + drivers — ONLY now that we know external
+        // entities exist. The dynamic import is deliberate (CLAUDE.md rule #8, category 3: build-tool
+        // startup deferral): the common codegen run has no external entities and must not pay the cost
+        // of loading pg/mongodb/the engine. The packages are declared deps; importing each triggers its
+        // @RegisterClass side effects so the ClassFactory can resolve the router + drivers below.
+        // snowflake-sdk is an optional peer the Snowflake driver loads only on connect.
+        await import('@memberjunction/external-data-sources');
+        await import('@memberjunction/external-data-source-postgres');
+        await import('@memberjunction/external-data-source-snowflake');
+        await import('@memberjunction/external-data-source-mongodb');
+        await import('@memberjunction/external-data-source-sqlserver');
+        await import('@memberjunction/external-data-source-mysql');
+        await import('@memberjunction/external-data-source-oracle');
+        const router = MJGlobal.Instance.ClassFactory.CreateInstance(ExternalDataSourceReadRouter);
+        if (!router) {
+            logError('   Cannot sync external entity fields: no ExternalDataSourceReadRouter is registered. Ensure @memberjunction/external-data-sources (and the relevant driver) are loaded in the CodeGen process.');
+            return { success: false, anyUpdates: false, relationshipsUpdated: false };
+        }
+        // Refresh the in-memory metadata cache so the external entities are present for EntityByName()
+        // inside manageSingleExternalEntity / manageSingleVirtualEntityField. External entities are
+        // often created/seeded after the CodeGen process loaded metadata at startup; without this
+        // refresh, EntityByName() returns null and field sync is SILENTLY skipped — the same gotcha
+        // already handled for config-created virtual entities above.
+        const md = new Metadata(); // global-provider-ok: codegen runs offline against a single provider
+        await md.Refresh();
+        for (const ee of externalEntities) {
+            const { success, updatedEntity, relationshipsUpdated: relUpdated } = await this.manageSingleExternalEntity(pool, ee, router, currentUser);
+            anyUpdates = anyUpdates || updatedEntity;
+            relationshipsUpdated = relationshipsUpdated || relUpdated;
+            if (!success) {
+                logError(`   Error managing external entity ${ee.Name}`);
+                bSuccess = false;
+            }
+        }
+        return { success: bSuccess, anyUpdates, relationshipsUpdated };
+    }
+    /**
+     * Whether an introspected remote object has columns we can safely sync. When false (object
+     * missing or zero columns), {@link manageSingleExternalEntity} skips the field sync rather than
+     * treating it as "the entity now has no fields" (which would DELETE every EntityField). Pure/testable.
+     */
+    externalObjectIsSyncable(obj) {
+        return !!(obj && Array.isArray(obj.Columns) && obj.Columns.length > 0);
+    }
+    /**
+     * Builds the DELETE for EntityFields no longer present in the freshly-introspected remote object,
+     * or '' when nothing should be removed. Field-name matching is case-insensitive. Pure/testable
+     * (no DB access) — the data-loss-sensitive computation guarded by {@link externalObjectIsSyncable}.
+     */
+    buildExternalFieldRemoveSQL(schema, existingFields, introspectedFieldNames) {
+        const present = new Set(introspectedFieldNames.map(n => n.trim().toLowerCase()));
+        const removeIds = existingFields.filter(f => !present.has(f.Name.trim().toLowerCase())).map(f => f.ID);
+        if (removeIds.length === 0) {
+            return '';
+        }
+        const idList = removeIds.map(id => `'${id}'`).join(',');
+        // Clear FK dependents (value-list entries) before the EntityField delete to avoid an FK violation,
+        // mirroring spDeleteUnneededEntityFields. Emitted as two statements separated by the platform terminator.
+        return `DELETE FROM ${this.qs(schema, 'EntityFieldValue')} WHERE EntityFieldID IN (${idList});\n` +
+            `DELETE FROM ${this.qs(schema, 'EntityField')} WHERE ID IN (${idList})`;
+    }
+    /**
+     * Introspects the remote schema for a single external entity and syncs its `EntityField` rows.
+     * Mirrors {@link manageSingleVirtualEntity} but sources the field list from the driver's
+     * `IntrospectSchema` (mapped to MJ types via {@link mapExternalNativeTypeToMJ}) instead of the
+     * local view's columns, and reuses {@link manageSingleVirtualEntityField} for create/update.
+     */
+    async manageSingleExternalEntity(pool, externalEntity, router, currentUser) {
+        let bSuccess = true;
+        let bUpdated = false;
+        let bRelationshipsUpdated = false;
+        // Introspection reaches a live remote system. A connection/auth/network failure there must NOT
+        // fail the whole CodeGen run (which also regenerates every MJ-DB entity) — treat it as a
+        // recoverable warning-skip, leaving this entity's existing fields untouched. Genuine errors in
+        // the field-sync logic below still fail (caught by the outer try/catch as success:false).
+        let descriptor;
+        try {
+            descriptor = await router.IntrospectExternalSchema(externalEntity.ExternalDataSourceID, externalEntity.SchemaName || undefined, currentUser);
+        }
+        catch (e) {
+            // Surface via logError (not logStatus) so a genuine driver bug isn't invisible in a green run —
+            // but still recover (success:true) so one unreachable remote source can't fail the whole run.
+            logError(`   ⚠️  External entity ${externalEntity.Name}: remote schema introspection failed (${e instanceof Error ? e.message : String(e)}) — skipping this entity (left untouched); CodeGen run continues.`);
+            return { success: true, updatedEntity: false, relationshipsUpdated: false };
+        }
+        try {
+            // Find the remote object backing this entity (prefer ExternalObjectName; fall back to
+            // BaseTable/Name). Match on bare or schema-qualified name, case-insensitively.
+            const target = (externalEntity.ExternalObjectName || externalEntity.BaseTable || externalEntity.Name || '').trim().toLowerCase();
+            const obj = descriptor.Objects.find(o => o.Name.trim().toLowerCase() === target ||
+                `${o.Schema ?? ''}.${o.Name}`.trim().toLowerCase() === target);
+            if (!obj) {
+                logStatus(`   ⚠️  External entity ${externalEntity.Name}: remote object '${externalEntity.ExternalObjectName ?? externalEntity.Name}' not found in introspected schema — skipping this entity (left untouched); CodeGen run continues.`);
+                // Recoverable skip (misconfigured/unreachable object): leave the entity untouched and
+                // continue the run rather than failing the whole metadata pass for one external entity.
+                return { success: true, updatedEntity: false, relationshipsUpdated: false };
+            }
+            // Guard the degenerate "found but zero columns" case the same way as "not found": a
+            // permission-limited / transient / partial introspection that returns an object shell with
+            // no columns must NOT be treated as "the entity now has no fields" — that would make the
+            // remove-list below every existing EntityField and DELETE them all (silent metadata loss).
+            // Bail with a warning instead and leave the existing fields untouched.
+            if (!this.externalObjectIsSyncable(obj)) {
+                logStatus(`   ⚠️  External entity ${externalEntity.Name}: remote object '${obj.Name}' introspected with zero columns — skipping field sync to avoid destroying existing field metadata; CodeGen run continues.`);
+                // Recoverable skip (permission-limited / transient / partial introspection): leave the
+                // existing fields untouched and continue, rather than failing the whole metadata pass.
+                return { success: true, updatedEntity: false, relationshipsUpdated: false };
+            }
+            // Map each introspected column into the veField shape that manageSingleVirtualEntityField consumes.
+            const eeFields = obj.Columns.map(c => {
+                const t = mapExternalNativeTypeToMJ(c.NativeType);
+                return { FieldName: c.Name, Type: t.Type, Length: t.Length, Precision: t.Precision, Scale: t.Scale, AllowsNull: c.Nullable, IsPrimaryKey: c.IsPrimaryKey };
+            });
+            const md = new Metadata(); // global-provider-ok: codegen runs offline against a single provider
+            const entity = md.EntityByName(externalEntity.Name);
+            if (entity) {
+                // remove EntityFields no longer present in the remote object
+                const sqlRemove = this.buildExternalFieldRemoveSQL(mj_core_schema(), entity.Fields, eeFields.map(ef => ef.FieldName));
+                if (sqlRemove) {
+                    await this.LogSQLAndExecute(pool, sqlRemove, `SQL text to remove fields from external entity ${externalEntity.Name}`);
+                    bUpdated = true;
+                }
+                // create/update each remote column. Use the real PK info from introspection; fall back
+                // to "first column is the PK" when the remote reported none (e.g. a view).
+                const anyPk = eeFields.some(f => f.IsPrimaryKey);
+                // How many columns make up the PK? Introspected PK count when present, else 1 (first-column
+                // fallback). A composite PK (>1) means no single column is unique on its own (M4).
+                const primaryKeyColumnCount = anyPk ? eeFields.filter(f => f.IsPrimaryKey).length : 1;
+                const singleColumnPrimaryKey = primaryKeyColumnCount === 1;
+                for (let i = 0; i < eeFields.length; i++) {
+                    const ef = eeFields[i];
+                    const makePrimaryKey = ef.IsPrimaryKey || (!anyPk && i === 0);
+                    // reconcilePrimaryKey=true: introspection is authoritative, so sync PK/Unique flags in BOTH
+                    // directions (set AND clear) — this clears a stale PK column when the remote PK moves (H5).
+                    const { success, updatedField } = await this.manageSingleVirtualEntityField(pool, externalEntity, ef, i + 1, makePrimaryKey, singleColumnPrimaryKey, true);
+                    bUpdated = bUpdated || updatedField;
+                    if (!success) {
+                        logError(`Error managing external entity field ${ef.FieldName} for external entity ${externalEntity.Name}`);
+                        bSuccess = false;
+                    }
+                }
+                // Baseline FK consumption: turn introspected single-column foreign keys into soft FKs
+                // (RelatedEntityID + IsSoftForeignKey) so the standard manageEntityRelationships pass
+                // materializes them into EntityRelationship records.
+                if (await this.manageExternalEntityRelationships(pool, externalEntity, obj, md)) {
+                    bUpdated = true;
+                    bRelationshipsUpdated = true;
+                }
+            }
+            if (bUpdated) {
+                const sqlUpdate = `UPDATE ${this.qs(mj_core_schema(), 'Entity')} SET ${this.qi(EntityInfo.UpdatedAtFieldName)}=${this.utcNow()} WHERE ID='${externalEntity.ID}'`;
+                await this.LogSQLAndExecute(pool, sqlUpdate, `SQL text to update external entity updated date for ${externalEntity.Name}`);
+            }
+            return { success: bSuccess, updatedEntity: bUpdated, relationshipsUpdated: bRelationshipsUpdated };
+        }
+        catch (e) {
+            logError(e instanceof Error ? e.message : String(e));
+            return { success: false, updatedEntity: bUpdated, relationshipsUpdated: bRelationshipsUpdated };
+        }
+    }
+    /**
+     * Baseline foreign-key consumption for an external entity: for each introspected single-column
+     * relationship whose referenced remote object is ALSO an imported external entity in the same
+     * data source, set the FK field's RelatedEntityID + RelatedEntityFieldName + IsSoftForeignKey=1.
+     * The standard `manageEntityRelationships` pass then materializes these into EntityRelationship
+     * records. Composite FKs and references to objects that aren't imported are skipped (logged) —
+     * those are the follow-up hardening. Returns true if any FK field was updated.
+     */
+    async manageExternalEntityRelationships(pool, externalEntity, obj, md) {
+        const relationships = obj.Relationships ?? [];
+        // No introspected relationships → nothing to set. We deliberately do NOT try to "clear dropped FKs"
+        // here: introspection returning zero relationships is AMBIGUOUS — it happens for drivers that don't
+        // report FK metadata at all (MongoDB), and for low-privilege logins where the catalog FK views are
+        // permission-filtered to zero rows (SQL Server sys.foreign_keys). Clearing on that signal would wipe
+        // user-authored soft-FKs (which are indistinguishable from CodeGen-set ones — there is no marker),
+        // so we leave existing FK metadata untouched. (See the change-detection below for the no-churn set path.)
+        if (relationships.length === 0) {
+            return false;
+        }
+        const dsID = externalEntity.ExternalDataSourceID;
+        const entity = md.EntityByName(externalEntity.Name); // read current field FK metadata (prior-run state)
+        const sqlStatements = [];
+        for (const rel of relationships) {
+            if (rel.Columns.length !== 1) {
+                logStatus(`      ⚠ external FK ${externalEntity.Name}.${rel.Name ?? '(unnamed)'}: composite key (${rel.Columns.length} columns) — skipped (baseline supports single-column FKs)`);
+                continue;
+            }
+            const { Column: fkColumn, ReferencedColumn: refColumn } = rel.Columns[0];
+            // Resolve the referenced object to an imported external entity in the SAME data source
+            // (match ExternalObjectName, falling back to BaseTable/Name — mirrors how this entity's
+            // own object is resolved above).
+            const target = rel.ReferencedObject.trim().toLowerCase();
+            const relatedEntity = md.Entities.find(e => e.ExternalDataSourceID && dsID && UUIDsEqual(e.ExternalDataSourceID, dsID) &&
+                ((e.ExternalObjectName || e.BaseTable || e.Name) || '').trim().toLowerCase() === target);
+            if (!relatedEntity) {
+                logStatus(`      ⚠ external FK ${externalEntity.Name}.${fkColumn} → '${rel.ReferencedObject}': referenced entity not imported — skipped`);
+                continue;
+            }
+            // Change detection (M5): skip the UPDATE when the field already carries the exact desired FK
+            // metadata — otherwise every CodeGen run re-UPDATEs the row, bumping __mj_UpdatedAt and logging
+            // a spurious change even though nothing changed.
+            const field = entity?.Fields.find(f => f.Name.trim().toLowerCase() === fkColumn.trim().toLowerCase());
+            const alreadySet = !!field && field.IsSoftForeignKey === true &&
+                !!field.RelatedEntityID && UUIDsEqual(field.RelatedEntityID, relatedEntity.ID) &&
+                (field.RelatedEntityFieldName ?? '').trim() === refColumn.trim();
+            if (alreadySet) {
+                continue;
+            }
+            sqlStatements.push(`UPDATE ${this.qs(mj_core_schema(), 'EntityField')}
+                         SET RelatedEntityID='${relatedEntity.ID}',
+                             RelatedEntityFieldName='${refColumn.replace(/'/g, "''")}',
+                             IsSoftForeignKey=1
+                         WHERE EntityID='${externalEntity.ID}' AND Name='${fkColumn.replace(/'/g, "''")}'`);
+            logStatus(`      ✓ external FK ${externalEntity.Name}.${fkColumn} → ${relatedEntity.Name}.${refColumn}`);
+        }
+        if (sqlStatements.length === 0) {
+            return false;
+        }
+        await this.LogSQLBatchAndExecute(pool, sqlStatements, `Set external foreign keys for ${externalEntity.Name}`);
+        return true;
+    }
     async manageSingleVirtualEntity(pool, virtualEntity) {
         let bSuccess = true;
         let bUpdated = false;
@@ -1132,9 +1418,40 @@ export class ManageMetadataBase {
             return { success: false, updatedEntity: bUpdated };
         }
     }
-    async manageSingleVirtualEntityField(pool, virtualEntity, veField, fieldSequence, makePrimaryKey) {
+    /**
+     * Desired PK/Unique flags for a virtual/external entity field. IsPrimaryKey mirrors makePrimaryKey;
+     * IsUnique is true ONLY for a SINGLE-column PK — a column of a COMPOSITE key is not unique on its own,
+     * so composite-key columns get IsUnique=false (M4). Pure function so the rule is unit-tested.
+     */
+    resolvePrimaryKeyFlags(makePrimaryKey, singleColumnPrimaryKey) {
+        return { wantPrimaryKey: makePrimaryKey, wantUnique: makePrimaryKey && singleColumnPrimaryKey };
+    }
+    /**
+     * Whether a field's PK/Unique flags need to change. In RECONCILE mode (external entities, whose
+     * introspected PK set is authoritative across ALL fields) we sync in BOTH directions — setting the
+     * flags when a column becomes a PK AND clearing them when it stops being one (H5: otherwise a stale PK
+     * column keeps IsPrimaryKey=1 when the remote PK moves, yielding duplicate PKs). In NON-reconcile mode
+     * (virtual entities, where makePrimaryKey is a one-time first-column bootstrap, not authoritative) we
+     * only SET on acquisition and never clear — avoiding wiping a legitimately-configured PK, and avoiding
+     * a spurious UPDATE (+ __mj_UpdatedAt bump) every run when nothing changed. Pure function; unit-tested.
+     */
+    primaryKeyFlagsChanged(current, want, makePrimaryKey, reconcile) {
+        if (!reconcile) {
+            return makePrimaryKey && !current.isPrimaryKey;
+        }
+        // Reconcile mode: sync PK status in both directions. IsUnique is managed ONLY in tandem with the
+        // primary key (when the field is, or is becoming, a PK) — we must NOT force IsUnique=0 on an
+        // ordinary non-PK column, which would silently wipe a legitimately-unique field (e.g. a user- or
+        // introspection-marked unique `email`) that has nothing to do with the primary key.
+        const pkStatusChanged = current.isPrimaryKey !== want.wantPrimaryKey;
+        const uniqueIsPkRelated = want.wantPrimaryKey || current.isPrimaryKey;
+        const uniqueChanged = uniqueIsPkRelated && current.isUnique !== want.wantUnique;
+        return pkStatusChanged || uniqueChanged;
+    }
+    async manageSingleVirtualEntityField(pool, virtualEntity, veField, fieldSequence, makePrimaryKey, singleColumnPrimaryKey = true, reconcilePrimaryKey = false) {
         // this protected checks to see if the field exists in the entity definition, and if not, adds it
         // if it exist it updates the entity field to match the view's data type and nullability attributes
+        const { wantPrimaryKey, wantUnique } = this.resolvePrimaryKeyFlags(makePrimaryKey, singleColumnPrimaryKey);
         // first, get the entity definition
         const md = new Metadata(); // global-provider-ok: codegen runs offline against a single provider
         const entity = md.EntityByName(virtualEntity.Name);
@@ -1143,8 +1460,15 @@ export class ManageMetadataBase {
         if (entity) {
             const field = entity.Fields.find(f => f.Name.trim().toLowerCase() === veField.FieldName.trim().toLowerCase());
             if (field) {
-                // have a match, so the field exists in the entity definition, now check to see if it needs to be updated
-                if (makePrimaryKey ||
+                // Decide whether the PK/Unique flags need to change. In RECONCILE mode (external entities,
+                // whose introspected PK set is authoritative across ALL fields) we sync in BOTH directions —
+                // setting the flags when a column becomes a PK AND clearing them when it stops being one (H5:
+                // otherwise a stale PK column keeps IsPrimaryKey=1 when the remote PK moves, yielding duplicate
+                // PKs). In NON-reconcile mode (virtual entities, where makePrimaryKey is a one-time first-column
+                // bootstrap, not authoritative) we only SET on acquisition and never clear, to avoid wiping a
+                // legitimately-configured PK — and we avoid re-UPDATING an unchanged row every run.
+                const pkFlagsChanged = this.primaryKeyFlagsChanged({ isPrimaryKey: field.IsPrimaryKey, isUnique: field.IsUnique }, { wantPrimaryKey, wantUnique }, makePrimaryKey, reconcilePrimaryKey);
+                if (pkFlagsChanged ||
                     field.Type.trim().toLowerCase() !== veField.Type.trim().toLowerCase() ||
                     field.Length !== veField.Length ||
                     field.AllowsNull !== veField.AllowsNull ||
@@ -1158,7 +1482,7 @@ export class ManageMetadataBase {
                                     Sequence=${fieldSequence},
                                     Type='${veField.Type}',
                                     AllowsNull=${this.boolLit(veField.AllowsNull)},
-                                    ${makePrimaryKey ? `IsPrimaryKey=${this.boolLit(true)},IsUnique=${this.boolLit(true)},` : ''}
+                                    ${pkFlagsChanged ? `IsPrimaryKey=${this.boolLit(wantPrimaryKey)},IsUnique=${this.boolLit(wantUnique)},` : ''}
                                     Length=${veField.Length},
                                     Precision=${veField.Precision},
                                     Scale=${veField.Scale}
@@ -1182,9 +1506,9 @@ export class ManageMetadataBase {
                                       ${q('Length')}, ${q('Precision')}, ${q('Scale')},
                                       ${q('Sequence')}, ${q('IsPrimaryKey')}, ${q('IsUnique')},
                                       ${q('__mj_CreatedAt')}, ${q('__mj_UpdatedAt')} )
-                            VALUES (  '${newEntityFieldUUID}', '${entity.ID}', '${veField.FieldName}', '${veField.Type}', ${this.boolLit(veField.AllowsNull)},
+                            VALUES (  '${newEntityFieldUUID}', '${entity.ID}', '${String(veField.FieldName).replace(/'/g, "''")}', '${veField.Type}', ${this.boolLit(veField.AllowsNull)},
                                        ${veField.Length}, ${veField.Precision}, ${veField.Scale},
-                                       ${safeSequence}, ${this.boolLit(makePrimaryKey)}, ${this.boolLit(makePrimaryKey)},
+                                       ${safeSequence}, ${this.boolLit(wantPrimaryKey)}, ${this.boolLit(wantUnique)},
                                        ${this.utcNow()}, ${this.utcNow()}
                                     )`;
                 await this.LogSQLAndExecute(pool, sqlAdd, `SQL text to add virtual entity field ${veField.FieldName} for entity ${virtualEntity.Name}`);
@@ -1959,6 +2283,17 @@ export class ManageMetadataBase {
             const entities = entitiesResult.recordset;
             if (entities && entities.length > 0) {
                 for (const e of entities) {
+                    // External-data-source entities have no physical MJ table by design, so they always
+                    // surface in vwEntitiesWithMissingBaseTables — but they must NOT be pruned (their data
+                    // lives on a remote system). Skip them here. This is the analogue of the VirtualEntity
+                    // exclusion, but kept as a single dialect-agnostic code guard rather than baked into the
+                    // per-dialect filter (getEntitiesWithMissingBaseTablesFilter) or the view, so it applies
+                    // uniformly across SQL Server and PostgreSQL. (The External Data Sources migration only
+                    // recreates the view to re-expose the ExternalDataSourceID column to this guard; it does
+                    // not exclude external entities at the SQL level.)
+                    if (e.ExternalDataSourceID) {
+                        continue;
+                    }
                     // for the given entity, wipe out the entity metadata and its core deps.
                     // the below could fail if there are non-core dependencies on the entity, but that's ok, we will flag that in the console
                     // for the admin to handle manually
@@ -2442,12 +2777,19 @@ export class ManageMetadataBase {
      */
     async ensureCreatedAtUpdatedAtFieldsExist(pool, excludeSchemas) {
         try {
+            // Only exclude external entities where the ExternalDataSourceID column exists (SQL Server today).
+            // On PostgreSQL the column is absent, so referencing it here would throw and fail CodeGen; external
+            // entities cannot exist there anyway, so the predicate is simply omitted.
+            const externalEntityClause = (await this.entityHasExternalDataSourceColumn(pool))
+                ? 'ExternalDataSourceID IS NULL AND -- external entities have no physical base table to add __mj_ system columns to (data is remote)'
+                : '';
             const sqlEntities = `SELECT
                                  *
                               FROM
                                  ${this.qs(mj_core_schema(), 'vwEntities')}
                               WHERE
                                  VirtualEntity = ${this.boolLit(false)} AND
+                                 ${externalEntityClause}
                                  TrackRecordChanges = ${this.boolLit(true)} AND
                                  SchemaName NOT IN (${excludeSchemas.map(s => `'${s}'`).join(',')})`;
             const entitiesResult = await this.runQuery(pool, sqlEntities);