@memberjunction/server 5.47.0 → 5.49.0

package/src/generic/RunViewResolver.ts

@@ -588,6 +588,12 @@ export class RunViewWithCacheCheckResultOutput {
 
   @Field(() => String, { nullable: true, description: 'Error message if status is error' })
   errorMessage?: string;
+
+  @Field(() => [AggregateResultOutput], {
+    nullable: true,
+    description: 'Aggregate results when status is stale and aggregates were requested (B40 — previously never marshalled, so CacheLocal callers got no aggregates at all)'
+  })
+  aggregateResults?: AggregateResultOutput[];
 }
 
 @ObjectType()
@@ -1073,6 +1079,9 @@ export class RunViewResolver extends ResolverBase {
           AuditLogDescription: item.params.AuditLogDescription,
           ResultType: (item.params.ResultType || 'simple') as 'simple' | 'entity_object' | 'count_only',
           StartRow: item.params.StartRow,
+          // Forward the aggregate request to the engine (B40) — omitted here as well as in the
+          // client's input map, so aggregates never reached InternalRunView on this transport.
+          Aggregates: item.params.Aggregates,
           AfterKey: item.params.AfterKey
             ? CompositeKey.FromKeyValuePairs(item.params.AfterKey.KeyValuePairs)
             : undefined,
@@ -1130,6 +1139,8 @@ export class RunViewResolver extends ResolverBase {
             maxUpdatedAt: result.maxUpdatedAt,
             rowCount: result.rowCount,
             errorMessage: result.errorMessage,
+            // Same serializer the regular RunView path uses (value → JSON string).
+            aggregateResults: this.processAggregateResults(result.aggregateResults),
           };
         }
 

package/src/index.ts

@@ -4,12 +4,13 @@ dotenv.config({ quiet: true });
 
 import { expressMiddleware } from '@as-integrations/express5';
 import { mergeSchemas } from '@graphql-tools/schema';
-import { Metadata, DatabasePlatform, SetProvider, StartupManager as StartupManagerImport, BaseEntity, BaseEntityEvent, RunView, DatabaseProviderBase } from '@memberjunction/core';
+import { Metadata, DatabasePlatform, SetProvider, StartupManager as StartupManagerImport, BaseEntity, BaseEntityEvent, RunView, DatabaseProviderBase, ResolveStartupMode } from '@memberjunction/core';
 import { resolveDbPlatformFromEnv } from '@memberjunction/generic-database-provider';
 import { MJGlobal, MJEventType, UUIDsEqual, ShutdownRegistry } from '@memberjunction/global';
 import { setupSQLServerClient, SQLServerDataProvider, SQLServerProviderConfigData, UserCache } from '@memberjunction/sqlserver-dataprovider';
 import { extendConnectionPoolWithQuery } from './util.js';
 import { registerIntegrationCustomColumnPromoter } from './integration/CustomColumnPromoter.js';
+import { DisableUnselectedEntityMaps, ReenableFieldMapsForEntityMap } from './integration/EntityMapLifecycle.js';
 import { default as BodyParser } from 'body-parser';
 import compression from 'compression'; // Add compression middleware
 import cors from 'cors';
@@ -26,6 +27,7 @@ import { BuildSchemaOptions, buildSchemaSync, GraphQLTimestamp, PubSubEngine } f
 import { PubSub } from 'graphql-subscriptions';
 import sql from 'mssql';
 import { WebSocketServer } from 'ws';
+import { RealtimeProxyServer } from './realtimeProxy/RealtimeProxyServer.js';
 import buildApolloServer from './apolloServer/index.js';
 import { configInfo, configFilePath, dbDatabase, dbHost, dbPort, dbUsername, graphqlPort, graphqlRootPath, mj_core_schema, websiteRunFromPackage, RESTApiOptions } from './config.js';
 import { default as jwt } from 'jsonwebtoken';
@@ -47,7 +49,7 @@ import { createTwilioTelephonyHandler, TWILIO_TELEPHONY_MOUNT_PATH, SetTwilioTel
 import { createVonageTelephonyHandler, VONAGE_TELEPHONY_MOUNT_PATH, SetVonageTelephonyService } from './telephony/index.js';
 import { RingCentralTelephonyService, SetRingCentralTelephonyService } from './telephony/index.js';
 import { createTeamsMeetingsHandler, TEAMS_MEETINGS_MOUNT_PATH, SetTeamsMeetingsService, GetTeamsMeetingsService, StartCalendarScheduler } from './telephony/index.js';
-import { InstallMediaUpgradeDispatcher } from './telephony/index.js';
+import { InstallMediaUpgradeDispatcher, IsGraphQLWsPath } from './telephony/index.js';
 
 import { resolve } from 'node:path';
 import { DataSourceInfo, raiseEvent } from './types.js';
@@ -323,10 +325,11 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
     // Refresh user cache using PostgreSQL
     await refreshUserCacheFromPG(pgPool, mj_core_schema);
 
-    // Run startup actions
+    // Run startup actions — same 'full' entry-point default as the SQL Server path
     const sysUser = UserCache.Instance.GetSystemUser();
     const backupSysUser = UserCache.Instance.Users.find(u => u.IsActive && u.Type === 'Owner');
-    await StartupManagerImport.Instance.Startup(false, sysUser || backupSysUser, provider);
+    const pgStartupMode = ResolveStartupMode({ configValue: configInfo.startup?.mode, defaultMode: 'full' });
+    await StartupManagerImport.Instance.Startup(false, sysUser || backupSysUser, provider, { mode: pgStartupMode.mode });
 
     // Monkey-patch SQLServerDataProvider.ExecuteSQLWithPool to support PostgreSQL
     // Generated resolvers call this static method with bracket-quoted SQL.
@@ -416,6 +419,17 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
           if (cgConfig.additionalSchemaInfo) {
             RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(cgConfig.additionalSchemaInfo);
             startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${cgConfig.additionalSchemaInfo}`);
+          } else if (RuntimeSchemaManager.Instance.IsEnabled) {
+            // U5 — boot-time assert: RSU is enabled but CodeGen has NO additionalSchemaInfo path
+            // configured, so RSU's soft-PK/FK writes land in a file CodeGen never reads. Every
+            // resolved soft PK would be silently lost ("No primary key found" per table). LOUD,
+            // named, and actionable — grep for RSU_ADDITIONAL_SCHEMA_INFO_DIVERGENCE.
+            console.error(
+              `[RSU_ADDITIONAL_SCHEMA_INFO_DIVERGENCE] Runtime Schema Update is ENABLED but mj.config.cjs has no ` +
+              `'additionalSchemaInfo' setting — RSU writes soft PK/FK definitions to '${process.env.RSU_ADDITIONAL_SCHEMA_INFO_PATH ?? 'additionalSchemaInfo.json'}' ` +
+              `while CodeGen reads none, so resolved soft primary keys will be silently LOST at codegen time. ` +
+              `Set 'additionalSchemaInfo' in mj.config.cjs to the same file so the write path and read path agree.`
+            );
           }
         } catch (codegenErr) {
           console.warn(`RSU in-process CodeGen runner setup failed (will fall back to child process): ${(codegenErr as Error).message}`);
@@ -468,7 +482,10 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
     }
 
     const config = new SQLServerProviderConfigData(pool, mj_core_schema, cacheRefreshInterval);
-    await setupSQLServerClient(config);
+    // MJAPI is a long-running server, so entry-point default is 'full' engine pre-warm;
+    // MJ_STARTUP_MODE / mj.config.cjs startup.mode can override per the shared precedence chain
+    const startupMode = ResolveStartupMode({ configValue: configInfo.startup?.mode, defaultMode: 'full' });
+    await setupSQLServerClient(config, { mode: startupMode.mode });
     lap('Metadata + Provider Setup', tPhase);
     startupLog.BeginPhase('Initializing data provider');
     const md = new Metadata(); // global-provider-ok: bootstrap
@@ -541,6 +558,14 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
           if (codegenConfig.additionalSchemaInfo) {
             RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(codegenConfig.additionalSchemaInfo);
             startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${codegenConfig.additionalSchemaInfo}`);
+          } else if (RuntimeSchemaManager.Instance.IsEnabled) {
+            // U5 — boot-time assert: write path ≠ read path (see the PostgreSQL branch for detail).
+            console.error(
+              `[RSU_ADDITIONAL_SCHEMA_INFO_DIVERGENCE] Runtime Schema Update is ENABLED but mj.config.cjs has no ` +
+              `'additionalSchemaInfo' setting — RSU writes soft PK/FK definitions to '${process.env.RSU_ADDITIONAL_SCHEMA_INFO_PATH ?? 'additionalSchemaInfo.json'}' ` +
+              `while CodeGen reads none, so resolved soft primary keys will be silently LOST at codegen time. ` +
+              `Set 'additionalSchemaInfo' in mj.config.cjs to the same file so the write path and read path agree.`
+            );
           }
         } catch (codegenErr) {
           console.warn(`RSU in-process CodeGen runner setup failed (will fall back to child process): ${(codegenErr as Error).message}`);
@@ -857,7 +882,34 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
 
   const httpServer = createServer(app);
 
-  const webSocketServer = new WebSocketServer({ server: httpServer, path: graphqlRootPath });
+  // `noServer` so we own the HTTP `upgrade` routing ourselves (below) and can dispatch by path between
+  // the graphql-ws server and the realtime proxy. (In `{ server, path }` mode, ws destroys any socket whose
+  // path doesn't match, which would kill the proxy's `/realtime-proxy` upgrades — hence the explicit router.)
+  const webSocketServer = new WebSocketServer({ noServer: true });
+
+  // Single upgrade router: the realtime proxy claims its own path first (an authenticated byte-tunnel that
+  // lets self-hosted realtime providers — e.g. HuggingFace speech-to-speech — run the shipped client-direct
+  // audio topology without the internal endpoint ever being exposed to the browser); everything on the
+  // graphql path goes to graphql-ws; anything else is rejected. Must be registered before httpServer.listen().
+  // Path acceptance (including the bare-root/`/graphql` alias) is centralized in `IsGraphQLWsPath` — see
+  // its doc comment for why the alias exists — so this listener and `InstallMediaUpgradeDispatcher`'s
+  // telephony-active replacement listener can never disagree on what counts as "the GraphQL path."
+  httpServer.on('upgrade', (request, socket, head) => {
+    if (RealtimeProxyServer.Instance.TryHandleUpgrade(request, socket, head as Buffer)) {
+      return;
+    }
+    let pathname = graphqlRootPath;
+    try {
+      pathname = new URL(request.url ?? graphqlRootPath, 'http://internal').pathname;
+    } catch {
+      /* unparseable — fall through to the graphql-path check, which will reject it */
+    }
+    if (IsGraphQLWsPath(pathname, graphqlRootPath)) {
+      webSocketServer.handleUpgrade(request, socket, head, (ws) => webSocketServer.emit('connection', ws, request));
+    } else {
+      socket.destroy();
+    }
+  });
 
   // Track per-connection expiry timers so we can clean them up on close
   const expiryTimers = new WeakMap<object, ReturnType<typeof setTimeout>>();
@@ -869,7 +921,12 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
       // This prevents re-validating (and re-logging) on every subscription message.
       onConnect: async (ctx) => {
         try {
-          const token = String(ctx.connectionParams?.Authorization);
+          // Only coerce a value that's actually present — `String(undefined)` yields the
+          // literal string "undefined" (truthy, 9 chars), which slips past getUserPayload's
+          // `!token` guard and fails later as 'Invalid token payload' (a full stack-trace log)
+          // instead of the quiet, routine 'Missing token' path. A WS client that connects
+          // before its auth handshake (extremely common) must land on the quiet path.
+          const token = ctx.connectionParams?.Authorization ? String(ctx.connectionParams.Authorization) : '';
           // Carry API keys from connectionParams so API-key / MCP / Node clients can authenticate the socket
           // (validated the same way as the HTTP x-mj-api-key / x-mj-user-api-key headers).
           const systemApiKey = ctx.connectionParams?.['x-mj-api-key'] ? String(ctx.connectionParams['x-mj-api-key']) : undefined;
@@ -1451,7 +1508,22 @@ async function processRSUPendingWork(): Promise<void> {
         let isNewMap = false;
 
         if (existingMapResult.Success && existingMapResult.Results.length > 0) {
-          entityMapID = existingMapResult.Results[0].ID;
+          const existingMap = existingMapResult.Results[0];
+          entityMapID = existingMap.ID;
+          // Re-add re-enables: a previously-removed (disabled) object that is
+          // re-selected comes back Active WITH its field maps re-enabled. Skipped in
+          // CreateDisabled (schema-evolution) mode — a refresh must never resurrect a
+          // map the user turned off.
+          if (!item.CreateDisabled && (existingMap.Status !== 'Active' || existingMap.SyncEnabled === false)) {
+            existingMap.Status = 'Active';
+            existingMap.SyncEnabled = true;
+            if (await existingMap.Save()) {
+              await ReenableFieldMapsForEntityMap(entityMapID, systemUser, Metadata.Provider); // global-provider-ok: server startup recovery — runs once before any per-request context exists
+              console.log(`[RSU] Re-enabled previously-disabled entity map for ${objName} → ${entity.Name} (${entityMapID})`);
+            } else {
+              console.warn(`[RSU] Failed to re-enable entity map for ${objName}: ${existingMap.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+            }
+          }
           console.log(`[RSU] Entity map already exists for ${objName} → ${entity.Name} (${entityMapID})`);
         } else {
           const entityMap = await md.GetEntityObject<MJCompanyIntegrationEntityMapEntity>(
@@ -1462,8 +1534,10 @@ async function processRSUPendingWork(): Promise<void> {
           entityMap.EntityID = entity.ID;
           entityMap.ExternalObjectName = objName;
           entityMap.SyncDirection = 'Pull';
-          entityMap.Status = 'Active';
-          entityMap.SyncEnabled = true;
+          // Refresh diff: schema-evolution-born maps arrive DISABLED (the user
+          // enables them after the refresh); the first-apply/selection path stays Active.
+          entityMap.Status = item.CreateDisabled ? 'Inactive' : 'Active';
+          entityMap.SyncEnabled = !item.CreateDisabled;
           const mapSaved = await entityMap.Save();
           if (!mapSaved) {
             console.warn(`[RSU] Failed to save entity map for ${objName}`);
@@ -1508,7 +1582,7 @@ async function processRSUPendingWork(): Promise<void> {
             fieldMap.IsKeyField = field.IsPrimaryKey ?? false;
             fieldMap.IsRequired = field.IsRequired ?? false;
             fieldMap.Direction = 'SourceToDest';
-            fieldMap.Status = 'Active';
+            fieldMap.Status = item.CreateDisabled ? 'Inactive' : 'Active';
             if (await fieldMap.Save()) fieldCount++;
           }
           console.log(`[RSU] Created entity map for ${objName} → ${entity.Name} with ${fieldCount} field maps${isNewMap ? '' : ' (existing map, new fields only)'}`);
@@ -1517,6 +1591,21 @@ async function processRSUPendingWork(): Promise<void> {
         }
       }
 
+      // Remove-as-disable: entity maps whose object is NOT in this apply's selection
+      // are disabled (data kept; re-selection re-enables). 'ignore' opts out for subset applies.
+      if ((item.UnselectedAction ?? 'disable') !== 'ignore') {
+        try {
+          const disabledObjects = await DisableUnselectedEntityMaps(
+            item.CompanyIntegrationID, item.SourceObjectNames, systemUser, Metadata.Provider // global-provider-ok: server startup recovery — runs once before any per-request context exists
+          );
+          if (disabledObjects.length > 0) {
+            console.log(`[RSU] Disabled ${disabledObjects.length} unselected entity map(s): ${disabledObjects.join(', ')}`);
+          }
+        } catch (disableErr) {
+          console.warn(`[RSU] Unselected-map disable failed: ${disableErr}`);
+        }
+      }
+
       // Start sync if requested
       if (item.StartSync !== false) {
         try {

package/src/integration/CustomColumnPromoter.ts

@@ -40,9 +40,26 @@ import {
     buildOverflowStats,
     planPromotions,
     sanitizeColumnName,
+    inferColumnTypeFromStats,
     type SchemaPromotionResult,
     type PromotionCandidate,
+    type CustomKeyStat,
+    type InferredColumnType,
 } from '@memberjunction/integration-engine';
+
+/**
+ * A custom-key candidate persisted to CompanyIntegration.Configuration.customKeyCandidates
+ * (out-of-band capture): key identity + sizing statistics + the inferred column
+ * type, NEVER raw sample values (PII-safe — the config row is operator-visible). Written
+ * with REPLACE semantics per synced entity on every sync, so keys that vanish evict.
+ */
+interface PersistedCustomKeyCandidate {
+    Key: string;
+    Occurrences: number;
+    TotalRecords: number;
+    MaxLength: number;
+    Inferred: InferredColumnType;
+}
 import type { BaseEntity } from '@memberjunction/core';
 import { DDLGenerator, type TargetColumnConfig, type DatabasePlatform } from '@memberjunction/integration-schema-builder';
 import { RuntimeSchemaManager, type RSUPipelineInput } from '@memberjunction/schema-engine';
@@ -82,6 +99,14 @@ export function registerIntegrationCustomColumnPromoter(): void {
     IntegrationEngine.Instance.SetPostSyncSchemaPromotionCallback(async (ctx) => {
         const user = ctx.ContextUser as UserInfo;
         const provider = ctx.Provider as IMetadataProvider | undefined;
+        const promoter = new IntegrationCustomColumnPromoter(user, provider);
+        // out-of-band candidates: persist the sync's in-memory custom-key statistics
+        // (REPLACE semantics per synced entity — vanished keys evict) so on-demand
+        // IntegrationListCustomColumnCandidates sees them even when the content-hash fast path
+        // skipped every row (the hash basis excludes overflow, so skipped rows never write it).
+        // Unconditional — runs whether or not auto-promote is on. PII-safe: names/lengths/types
+        // only, never raw values.
+        await promoter.PersistCandidateStats(ctx.CompanyIntegrationID, ctx.SyncedEntityNames, ctx.CustomKeyStats);
         // GATE — promotion runs RSU (ADD COLUMN + register EntityField) + restart, which is disruptive.
         // It is OPT-IN per connection (DEFAULT OFF): by default a sync only CAPTURES unmapped fields into
         // the overflow column; the user triggers promotion on demand via IntegrationPromoteCustomColumns
@@ -94,8 +119,7 @@ export function registerIntegrationCustomColumnPromoter(): void {
             );
             return { Promoted: false, ColumnsAdded: [], SchemaUpdatePending: false };
         }
-        const promoter = new IntegrationCustomColumnPromoter(user, provider);
-        return promoter.PromoteForSync(ctx.CompanyIntegrationID, ctx.SyncedEntityNames);
+        return promoter.PromoteForSync(ctx.CompanyIntegrationID, ctx.SyncedEntityNames, ctx.CustomKeyStats);
     });
     // Verbose-only: this is a boot-time registration confirmation, not operator-actionable at
     // standard level. Routes through the global verbose gate (set from the server's telemetry.level).
@@ -136,6 +160,7 @@ export class IntegrationCustomColumnPromoter {
     public async PromoteForSync(
         companyIntegrationID: string,
         syncedEntityNames: string[],
+        customKeyStats?: Record<string, CustomKeyStat[]>,
     ): Promise<SchemaPromotionResult> {
         const integrationID = await this.resolveIntegrationID(companyIntegrationID);
         if (!integrationID) return NOT_PROMOTED;
@@ -143,7 +168,7 @@ export class IntegrationCustomColumnPromoter {
         const columnsAdded: Array<{ EntityName: string; ColumnName: string }> = [];
         for (const entityName of syncedEntityNames) {
             try {
-                const added = await this.promoteEntity(companyIntegrationID, integrationID, entityName);
+                const added = await this.promoteEntity(companyIntegrationID, integrationID, entityName, customKeyStats?.[entityName]);
                 columnsAdded.push(...added);
             } catch (err) {
                 // One entity's promotion failure must not abort the others, and never the sync.
@@ -164,8 +189,9 @@ export class IntegrationCustomColumnPromoter {
         companyIntegrationID: string,
         integrationID: string,
         entityName: string,
+        inRunStats?: CustomKeyStat[],
     ): Promise<Array<{ EntityName: string; ColumnName: string }>> {
-        const planned = await this.planWorkForEntity(companyIntegrationID, entityName);
+        const planned = await this.planWorkForEntity(companyIntegrationID, entityName, inRunStats);
         if (!planned || planned.work.length === 0) return []; // no overflow / no entity map / already converged
         const { entityInfo, entityMap } = planned;
         let work = planned.work;
@@ -208,14 +234,19 @@ export class IntegrationCustomColumnPromoter {
     private async planWorkForEntity(
         companyIntegrationID: string,
         entityName: string,
+        inRunStats?: CustomKeyStat[],
     ): Promise<{ entityInfo: EntityInfo; entityMap: { ID: string; ExternalObjectName: string }; work: WorkItem[] } | null> {
         const entityInfo = this.provider.EntityByName(entityName);
         if (!entityInfo?.SchemaName || !entityInfo.BaseTable) return null;
         // No overflow column on this table (predates the feature) → nothing to promote.
         if (!entityInfo.Fields.some(f => f.Name === CUSTOM_OVERFLOW_COLUMN)) return null;
 
+        // Candidate sources, most-authoritative first (dedup by key):
+        //  1. live overflow-column scan (rows that were actually written),
+        // 2. THIS run's in-memory stats (rows the content-hash fast path skipped —
+        //     out-of-band capture; the hash basis excludes overflow so skips never write it),
+        //  3. candidates persisted from prior runs (survive restarts for on-demand listing).
         const overflowJson = await this.scanOverflow(entityName);
-        if (overflowJson.length === 0) return null; // no customs captured
 
         // U3 note (rkihm-BC review, #3061): this in-repo promotion path passes no `LockUntilFullSync`, so it
         // does NOT yet enforce "hold promotion until a full sync since the last schema change." The lever
@@ -223,7 +254,11 @@ export class IntegrationCustomColumnPromoter {
         // sync has completed post-rediscovery — DEFERRED and tracked as a follow-up. The engine ships the
         // gate; wiring MJServer's own consumer to set it is a separate change. Until then, this path retains
         // the pre-U3 behavior for a rediscover-then-incremental sequence.
-        const passing = planPromotions(buildOverflowStats(overflowJson), {});
+        const byKey = new Map<string, PromotionCandidate>();
+        for (const c of planPromotions(buildOverflowStats(overflowJson), {})) byKey.set(c.Key, c);
+        for (const c of this.candidatesFromStats(inRunStats)) if (!byKey.has(c.Key)) byKey.set(c.Key, c);
+        for (const c of await this.loadPersistedCandidates(companyIntegrationID, entityName)) if (!byKey.has(c.Key)) byKey.set(c.Key, c);
+        const passing = [...byKey.values()];
         if (passing.length === 0) return null;
 
         const entityMap = await this.findEntityMap(companyIntegrationID, entityName);
@@ -258,6 +293,88 @@ export class IntegrationCustomColumnPromoter {
         }));
     }
 
+    /** Maps THIS run's in-memory custom-key stats to promotion candidates (out-of-band capture). */
+    private candidatesFromStats(stats?: CustomKeyStat[]): PromotionCandidate[] {
+        if (!stats || stats.length === 0) return [];
+        return stats
+            .filter(s => s.TotalRecords > 0 && s.Occurrences > 0)
+            .map(s => ({
+                Key: s.Key,
+                Coverage: s.Occurrences / s.TotalRecords,
+                Inferred: inferColumnTypeFromStats(s.SampleValues, s.MaxLength),
+            }));
+    }
+
+    /** Loads candidates persisted by {@link PersistCandidateStats} for one entity (empty on any gap). */
+    private async loadPersistedCandidates(companyIntegrationID: string, entityName: string): Promise<PromotionCandidate[]> {
+        try {
+            const ci = await this.provider.GetEntityObject<MJCompanyIntegrationEntity>('MJ: Company Integrations', this.user);
+            if (!(await ci.Load(companyIntegrationID)) || !ci.Configuration) return [];
+            const cfg = JSON.parse(ci.Configuration) as { customKeyCandidates?: Record<string, PersistedCustomKeyCandidate[]> };
+            const persisted = cfg.customKeyCandidates?.[entityName];
+            if (!Array.isArray(persisted)) return [];
+            return persisted
+                .filter(p => p && typeof p.Key === 'string' && p.TotalRecords > 0)
+                .map(p => ({ Key: p.Key, Coverage: p.Occurrences / p.TotalRecords, Inferred: p.Inferred }));
+        } catch {
+            return [];
+        }
+    }
+
+    /**
+     * Persists the sync's custom-key statistics onto CompanyIntegration.Configuration
+     * (`customKeyCandidates`, keyed by entity name) with REPLACE semantics per SYNCED entity:
+     * an entity synced with stats gets exactly this run's key set (vanished keys evict); an
+     * entity synced with NO custom keys has its entry removed. Unsynced entities keep theirs.
+     * PII-safe — key names, counts, lengths and inferred types only, never raw values. Caps
+     * the stored set per entity so a pathological source can't bloat the config row.
+     */
+    public async PersistCandidateStats(
+        companyIntegrationID: string,
+        syncedEntityNames: string[],
+        statsByEntity?: Record<string, CustomKeyStat[]>,
+    ): Promise<void> {
+        const MAX_PERSISTED_KEYS_PER_ENTITY = 200;
+        try {
+            const ci = await this.provider.GetEntityObject<MJCompanyIntegrationEntity>('MJ: Company Integrations', this.user);
+            if (!(await ci.Load(companyIntegrationID))) return;
+            const cfg = (ci.Configuration ? JSON.parse(ci.Configuration) : {}) as Record<string, unknown> & {
+                customKeyCandidates?: Record<string, PersistedCustomKeyCandidate[]>;
+            };
+            const candidates = { ...(cfg.customKeyCandidates ?? {}) };
+            let changed = false;
+            for (const entityName of syncedEntityNames) {
+                const stats = statsByEntity?.[entityName];
+                if (stats && stats.length > 0) {
+                    candidates[entityName] = stats
+                        .filter(s => s.TotalRecords > 0 && s.Occurrences > 0)
+                        .slice(0, MAX_PERSISTED_KEYS_PER_ENTITY)
+                        .map(s => ({
+                            Key: s.Key,
+                            Occurrences: s.Occurrences,
+                            TotalRecords: s.TotalRecords,
+                            MaxLength: s.MaxLength,
+                            Inferred: inferColumnTypeFromStats(s.SampleValues, s.MaxLength),
+                        }));
+                    changed = true;
+                } else if (candidates[entityName]) {
+                    delete candidates[entityName]; // synced clean → evict stale candidates
+                    changed = true;
+                }
+            }
+            if (!changed) return;
+            if (Object.keys(candidates).length > 0) cfg.customKeyCandidates = candidates;
+            else delete cfg.customKeyCandidates;
+            ci.Configuration = JSON.stringify(cfg);
+            if (!await ci.Save()) {
+                LogError(`[CustomColumnPromoter] Failed to persist custom-key candidates for CI ${companyIntegrationID}: ${ci.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+            }
+        } catch (err) {
+            // Best-effort — candidate persistence must never fail the sync.
+            LogError(`[CustomColumnPromoter] PersistCandidateStats failed for CI ${companyIntegrationID}: ${this.msg(err)}`);
+        }
+    }
+
     /** Samples the overflow column (rows where it is non-null) — dialect-agnostic via RunView. */
     private async scanOverflow(entityName: string): Promise<Array<string | null>> {
         const rv = new RunView();

package/src/integration/EntityMapLifecycle.ts

@@ -0,0 +1,221 @@
+/**
+ * @fileoverview Entity-map / field-map lifecycle helpers shared by the ApplyAll selection
+ * path, the post-restart RSU pending-work consumer, and the schema-evolution refresh diff
+ * Selection removal DISABLES (never deletes — data is kept), re-selection
+ * re-enables, and a refresh's new objects arrive disabled-by-default.
+ */
+import { CompositeKey, IMetadataProvider, LogError, RunView, UserInfo } from '@memberjunction/core';
+import type {
+    MJCompanyIntegrationEntityMapEntity,
+    MJCompanyIntegrationFieldMapEntity,
+    MJCompanyIntegrationSyncWatermarkEntity,
+} from '@memberjunction/core-entities';
+
+/**
+ * Remove-as-disable: entity maps for this CompanyIntegration whose
+ * ExternalObjectName is NOT in the selected set are set `Status='Inactive'` +
+ * `SyncEnabled=false`, and their Active field maps are disabled with them. Data and
+ * tables are KEPT — a later re-selection re-enables everything (see
+ * {@link ReenableFieldMapsForEntityMap} + the ApplyAll create-or-reuse path).
+ *
+ * @returns the ExternalObjectNames actually disabled this call.
+ */
+export async function DisableUnselectedEntityMaps(
+    companyIntegrationID: string,
+    selectedExternalObjectNames: string[],
+    contextUser: UserInfo,
+    md: IMetadataProvider,
+): Promise<string[]> {
+    const selected = new Set(selectedExternalObjectNames.map(n => n.toLowerCase()));
+    const disabled: string[] = [];
+
+    const maps = await new RunView().RunView<MJCompanyIntegrationEntityMapEntity>({
+        EntityName: 'MJ: Company Integration Entity Maps',
+        ExtraFilter: `CompanyIntegrationID='${companyIntegrationID.replace(/'/g, "''")}'`,
+        ResultType: 'entity_object',
+        BypassCache: true, // lifecycle decisions must read committed state
+    }, contextUser);
+    if (!maps.Success) return disabled;
+
+    for (const em of maps.Results) {
+        const name = (em.ExternalObjectName ?? '').toLowerCase();
+        if (!name || selected.has(name)) continue;
+        if (em.Status === 'Inactive' && em.SyncEnabled === false) continue; // already off
+        em.Status = 'Inactive';
+        em.SyncEnabled = false;
+        if (!await em.Save()) {
+            LogError(`[EntityMapLifecycle] Failed to disable entity map for '${em.ExternalObjectName}': ${em.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+            continue;
+        }
+        await SetFieldMapsStatus(em.ID, 'Inactive', contextUser, md);
+        disabled.push(em.ExternalObjectName ?? em.ID);
+    }
+    return disabled;
+}
+
+/**
+ * Sets one entity map's enabled state (Status + SyncEnabled) and cascades the same status to
+ * its field maps. Used by the schema-evolution diff: NEW objects arrive
+ * `enabled=false` by default (the user turns them on after the refresh; autoEnableNewObjects
+ * opts into enabled=true).
+ */
+export async function SetEntityMapEnabled(
+    entityMapID: string,
+    enabled: boolean,
+    contextUser: UserInfo,
+    md: IMetadataProvider,
+): Promise<boolean> {
+    const em = await md.GetEntityObject<MJCompanyIntegrationEntityMapEntity>('MJ: Company Integration Entity Maps', contextUser);
+    if (!await em.InnerLoad(CompositeKey.FromID(entityMapID))) return false;
+    em.Status = enabled ? 'Active' : 'Inactive';
+    em.SyncEnabled = enabled;
+    if (!await em.Save()) {
+        LogError(`[EntityMapLifecycle] Failed to set entity map ${entityMapID} enabled=${enabled}: ${em.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+        return false;
+    }
+    await SetFieldMapsStatus(entityMapID, enabled ? 'Active' : 'Inactive', contextUser, md);
+    return true;
+}
+
+/**
+ * Re-add re-enables: flips every non-Active field map of an entity map back to
+ * Active. Called when a previously-removed (disabled) object is re-selected — the entity
+ * map itself is re-activated by the caller's create-or-reuse; this brings its field maps
+ * back with it.
+ */
+export async function ReenableFieldMapsForEntityMap(
+    entityMapID: string,
+    contextUser: UserInfo,
+    md: IMetadataProvider,
+): Promise<number> {
+    return SetFieldMapsStatus(entityMapID, 'Active', contextUser, md);
+}
+
+/** Sets every field map of an entity map to the given status (skips rows already there). */
+async function SetFieldMapsStatus(
+    entityMapID: string,
+    status: 'Active' | 'Inactive',
+    contextUser: UserInfo,
+    md: IMetadataProvider,
+): Promise<number> {
+    const fms = await new RunView().RunView<MJCompanyIntegrationFieldMapEntity>({
+        EntityName: 'MJ: Company Integration Field Maps',
+        ExtraFilter: `EntityMapID='${entityMapID.replace(/'/g, "''")}'`,
+        ResultType: 'entity_object',
+        BypassCache: true,
+    }, contextUser);
+    if (!fms.Success) return 0;
+    let changed = 0;
+    for (const fm of fms.Results) {
+        if (fm.Status === status) continue;
+        fm.Status = status;
+        if (await fm.Save()) changed++;
+        else LogError(`[EntityMapLifecycle] Failed to set field map '${fm.SourceFieldName}' to ${status}: ${fm.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+    }
+    return changed;
+}
+
+/**
+ * DAG-aware removal advisory (rsuplan "adding and removing tables"): when a removal
+ * disables object X but a STILL-ACTIVE object depends on X (hard-FK or soft-FK parent edge —
+ * the same edges the sync engine layers by), the dependents are NOT auto-removed; instead each
+ * broken dependency is surfaced as a warning so a consumer can prompt the user to force-remove
+ * (or re-add) them. Pure + deterministic for unit testing: callers supply the parent map.
+ *
+ * @param activeObjectNames  objects remaining active after the removal
+ * @param removedObjectNames objects being removed/disabled this pass
+ * @param parentsByLowerName object name (lowercased) → its parent object names (the DAG edges)
+ * @returns one warning per active object that depends on ≥1 removed object
+ */
+export function ComputeRemovedDependencyWarnings(
+    activeObjectNames: string[],
+    removedObjectNames: string[],
+    parentsByLowerName: Map<string, string[]>,
+): string[] {
+    if (removedObjectNames.length === 0 || activeObjectNames.length === 0) return [];
+    const removedLower = new Set(removedObjectNames.map(n => n.toLowerCase()));
+    const warnings: string[] = [];
+    for (const name of activeObjectNames) {
+        const parents = parentsByLowerName.get(name.toLowerCase()) ?? [];
+        const cut = parents.filter(p => removedLower.has(p.toLowerCase()));
+        if (cut.length > 0) {
+            warnings.push(
+                `Object '${name}' remains active but depends on removed object(s) ${cut.map(c => `'${c}'`).join(', ')} — ` +
+                `its parent references cannot resolve until they return; review whether '${name}' should be removed too (force-remove) or the parent re-added.`
+            );
+        }
+    }
+    return warnings;
+}
+
+/**
+ * DAG-aware FORCE-REMOVE (rsuplan "adding and removing tables": "a consumer may want the user
+ * to force remove those too"): computes the TRANSITIVE closure of still-active objects that
+ * depend — directly or through other dependents — on a removed object. Callers opting into
+ * cascade removal disable this set alongside the removed objects (still remove-as-disable,
+ * never delete; a re-add re-enables). Pure + deterministic for unit testing.
+ *
+ * @param activeObjectNames  objects remaining active after the (non-cascaded) removal
+ * @param removedObjectNames objects being removed/disabled this pass
+ * @param parentsByLowerName object name (lowercased) → its parent object names (the DAG edges)
+ * @returns the active objects to cascade-disable, in dependency-discovery order
+ */
+export function ComputeCascadeRemovalSet(
+    activeObjectNames: string[],
+    removedObjectNames: string[],
+    parentsByLowerName: Map<string, string[]>,
+): string[] {
+    if (removedObjectNames.length === 0 || activeObjectNames.length === 0) return [];
+    const gone = new Set(removedObjectNames.map(n => n.toLowerCase()));
+    const cascade: string[] = [];
+    let grew = true;
+    while (grew) {
+        grew = false;
+        for (const name of activeObjectNames) {
+            const lower = name.toLowerCase();
+            if (gone.has(lower)) continue; // already removed or already cascaded
+            const parents = parentsByLowerName.get(lower) ?? [];
+            if (parents.some(p => gone.has(p.toLowerCase()))) {
+                gone.add(lower);
+                cascade.push(name);
+                grew = true;
+            }
+        }
+    }
+    return cascade;
+}
+
+/**
+ * U10 schema-change: resets the PULL watermark for the given entity maps so the
+ * next sync ignores the incremental cursor, re-fetches the full object, and BACKFILLS rows
+ * for newly-added columns (content-hash keeps genuinely-unchanged mapped values write-free
+ * only when the mapped set is unchanged — a schema change alters the mapped set, so changed
+ * rows rewrite exactly once). Nulling the value (not deleting the row) keeps bookkeeping.
+ */
+export async function ResetPullWatermarks(
+    entityMapIDs: string[],
+    contextUser: UserInfo,
+    md: IMetadataProvider,
+): Promise<string[]> {
+    const reset: string[] = [];
+    if (entityMapIDs.length === 0) return reset;
+    const inList = entityMapIDs.map(id => `'${id.replace(/'/g, "''")}'`).join(',');
+    const wms = await new RunView().RunView<{ ID: string; EntityMapID: string }>({
+        EntityName: 'MJ: Company Integration Sync Watermarks',
+        ExtraFilter: `EntityMapID IN (${inList}) AND Direction='Pull'`,
+        Fields: ['ID', 'EntityMapID'],
+        ResultType: 'simple',
+        BypassCache: true,
+    }, contextUser);
+    if (!wms.Success) return reset;
+    for (const row of wms.Results) {
+        const wm = await md.GetEntityObject<MJCompanyIntegrationSyncWatermarkEntity>(
+            'MJ: Company Integration Sync Watermarks', contextUser
+        );
+        if (!await wm.InnerLoad(CompositeKey.FromID(row.ID))) continue;
+        wm.WatermarkValue = null;
+        if (await wm.Save()) reset.push(row.EntityMapID);
+        else LogError(`[EntityMapLifecycle] Failed to reset watermark for entity map ${row.EntityMapID}: ${wm.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+    }
+    return reset;
+}