@memberjunction/server 5.48.0 → 5.50.0

package/src/resolvers/IntegrationDiscoveryResolver.ts

@@ -1,6 +1,7 @@
 import { Resolver, Query, Mutation, Arg, Ctx, ObjectType, Field, InputType, Int, Float } from "type-graphql";
 import { CompositeKey, DatabaseProviderBase, EntityInfo, LocalCacheManager, Metadata, RunView, UserInfo, LogError, LogStatus, IMetadataProvider, TransactionGroupBase } from "@memberjunction/core";
 import { GetReadOnlyProvider, GetReadWriteProvider } from "../util.js";
+import { UUIDsEqual } from "@memberjunction/global";
 import { CronExpressionHelper } from "@memberjunction/scheduling-engine";
 import {
     MJCompanyIntegrationEntity,
@@ -52,6 +53,7 @@ import { IntegrationProgressReader } from "@memberjunction/integration-progress-
 import type { IntegrationRunSnapshot, IntegrationRunKind } from "@memberjunction/integration-progress-artifacts";
 import { ResolverBase } from "../generic/ResolverBase.js";
 import { IntegrationCustomColumnPromoter } from "../integration/CustomColumnPromoter.js";
+import { ComputeCascadeRemovalSet, ComputeRemovedDependencyWarnings, DisableUnselectedEntityMaps, ReenableFieldMapsForEntityMap, ResetPullWatermarks, SetEntityMapEnabled } from "../integration/EntityMapLifecycle.js";
 import { AppContext } from "../types.js";
 import { RequireSystemUser } from "../directives/RequireSystemUser.js";
 import { UserCache } from "@memberjunction/sqlserver-dataprovider";
@@ -64,6 +66,10 @@ class RSUStepOutput {
     @Field() Status: string;
     @Field() DurationMs: number;
     @Field() Message: string;
+    /** U11 — 1-based position within the pipeline's expected step sequence (determinate stepper). */
+    @Field(() => Int, { nullable: true }) StepIndex?: number;
+    /** U11 — expected total steps for the run. */
+    @Field(() => Int, { nullable: true }) StepTotal?: number;
 }
 
 @ObjectType()
@@ -100,9 +106,10 @@ class ApplyAllInput {
     @Field({ nullable: true }) CronExpression?: string;
     @Field({ nullable: true }) ScheduleTimezone?: string;
     @Field(() => Boolean, { nullable: true, defaultValue: true, description: 'If false, skips the sync step after schema + entity maps are created' }) StartSync?: boolean;
-    @Field(() => Boolean, { nullable: true, defaultValue: false, description: 'If true, ignores watermarks and does a full re-fetch' }) FullSync?: boolean;
+    @Field(() => Boolean, { nullable: true, description: 'If true, ignores watermarks and does a full re-fetch. When OMITTED: defaults to TRUE on the connector\'s FIRST apply (no prior entity maps — a full sync by default for the first sync), FALSE thereafter.' }) FullSync?: boolean;
     @Field({ nullable: true, defaultValue: 'created', description: 'Sync scope: "created" = only newly created entity maps, "all" = all maps for the connector' }) SyncScope?: string;
     @Field({ nullable: true, defaultValue: 'Pull', description: 'SyncDirection applied to all created entity maps: Pull | Push | Bidirectional. Defaults to Pull.' }) DefaultSyncDirection?: string;
+    @Field({ nullable: true, defaultValue: 'disable', description: 'Remove-as-disable: what to do with existing entity maps whose object is NOT in this selection. "disable" (default) = Status=Disabled + SyncEnabled=false (+ field maps disabled; data kept, re-selection re-enables). "ignore" = leave them untouched (additive apply — use for subset/partial applies).' }) UnselectedAction?: string;
 }
 
 @ObjectType()
@@ -164,6 +171,7 @@ class ApplyAllBatchInput {
     @Field({ nullable: true, defaultValue: 'created', description: 'Sync scope: "created" = only newly created entity maps, "all" = all maps for the connector' }) SyncScope?: string;
     @Field({ nullable: true, description: 'Override sync direction for the initial sync: Pull | Push | Bidirectional. Defaults to entity map SyncDirection.' }) SyncDirection?: string;
     @Field({ nullable: true, description: 'Override sync direction stored in the created schedule: Pull | Push | Bidirectional.' }) ScheduleSyncDirection?: string;
+    @Field({ nullable: true, defaultValue: 'disable', description: 'Remove-as-disable (same as ApplyAllInput.UnselectedAction): "disable" (default) disables entity maps whose object is absent from each connector\'s selection; "ignore" leaves them untouched.' }) UnselectedAction?: string;
 }
 
 @ObjectType()
@@ -215,6 +223,16 @@ class SchemaEvolutionOutput {
     @Field({ nullable: true }) GitCommitSuccess?: boolean;
     @Field({ nullable: true }) APIRestarted?: boolean;
     @Field(() => [String], { nullable: true }) Warnings?: string[];
+    /** Refresh diff: objects newly present in this resolution (tables created; EM/EFMs created DISABLED unless autoEnableNewObjects). */
+    @Field(() => [String], { nullable: true }) NewObjects?: string[];
+    /** Refresh diff: objects absent from this resolution — their EM/EFMs were disabled (data kept). */
+    @Field(() => [String], { nullable: true }) RemovedObjects?: string[];
+    /** DAG cascade (cascadeRemoveDependents=true): still-active dependents of removed objects that were force-disabled with them (transitive closure; data kept, re-add re-enables). */
+    @Field(() => [String], { nullable: true }) CascadeRemovedObjects?: string[];
+    /** Continuing objects whose column set / constraints changed this run. */
+    @Field(() => [String], { nullable: true }) ChangedObjects?: string[];
+    /** U10 — objects whose Pull watermark was reset so the next sync backfills the schema change. */
+    @Field(() => [String], { nullable: true }) WatermarksReset?: string[];
 }
 
 // ─── Connector Capabilities Output Type ─────────────────────────────────────
@@ -829,6 +847,10 @@ class OperationProgressOutput {
     @Field({ nullable: true }) RecordsErrored?: number;
     @Field({ nullable: true }) RSUStep?: string;
     @Field({ nullable: true }) RSURunning?: boolean;
+    /** U11 — 1-based index of the current RSU step (determinate stepper). */
+    @Field(() => Int, { nullable: true }) RSUStepIndex?: number;
+    /** U11 — expected total RSU steps for the in-flight run. */
+    @Field(() => Int, { nullable: true }) RSUStepTotal?: number;
     @Field({ nullable: true }) ElapsedMs?: number;
     @Field({ nullable: true }) StartedAt?: string;
 }
@@ -979,6 +1001,14 @@ class ListSourceObjectsItem {
     @Field({ nullable: true }) IntegrationObjectID?: string;
     /** True when the source system flags this as user/custom (e.g. SF __c names). */
     @Field() IsCustom: boolean;
+    /**
+     * DAG exposure: the source object names this object DEPENDS ON (its parents in
+     * the sync DAG — hard FK edges via RelatedIntegrationObjectID plus soft-FK declarations
+     * via Configuration.parentObjectName(s)/ReferencedType). Lets a selection UI auto-check
+     * dependencies when this object is picked and warn when a depended-on object is removed.
+     * Empty for root objects; only populated for objects with a persisted IntegrationObject row.
+     */
+    @Field(() => [String], { nullable: true }) DependsOn?: string[];
 }
 
 @ObjectType()
@@ -1117,8 +1147,13 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                     SupportsWrite: true,
                   }) as ExternalObjectSchema);
 
+            // DAG exposure: per-object parent names from the same FK graph the sync
+            // engine layers by, so a consumer can auto-check dependencies at selection time.
+            const depsByName = this.computeObjectDependencyNames(companyIntegration.IntegrationID);
+
             const merged: ListSourceObjectsItem[] = sourceObjects.map(o => {
                 const existing = existingByName.get(o.Name);
+                const dependsOn = depsByName.get(o.Name.toLowerCase());
                 return {
                     Name: o.Name,
                     Label: o.Label,
@@ -1128,6 +1163,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                     AlreadyPersisted: existing != null,
                     IntegrationObjectID: existing?.ID,
                     IsCustom: this.isCustomObjectName(o.Name, existing?.IsCustom),
+                    DependsOn: dependsOn && dependsOn.length > 0 ? dependsOn : undefined,
                 };
             });
             merged.sort((a, b) => a.Name.localeCompare(b.Name));
@@ -1174,6 +1210,50 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         return name.endsWith('__c');
     }
 
+    /**
+     * DAG exposure: object name (lowercased) → sorted parent object names, derived from
+     * the SAME dependency edges the sync engine layers by — hard FK pointers
+     * (IntegrationObjectField.RelatedIntegrationObjectID) plus soft-FK declarations in the IO's
+     * Configuration (parentObjectName / parentObjectNames map / ReferencedType). Reads the
+     * engine cache only (no DB round-trip); objects with no persisted IO row have no entry.
+     */
+    private computeObjectDependencyNames(integrationID: string): Map<string, string[]> {
+        const engine = IntegrationEngineBase.Instance;
+        const ios = engine.GetIntegrationObjectsByIntegrationID(integrationID);
+        const nameById = new Map(ios.map(io => [io.ID.toUpperCase(), io.Name] as const));
+        const byLowerName = new Map(ios.map(io => [io.Name.toLowerCase(), io] as const));
+        const deps = new Map<string, string[]>();
+        for (const io of ios) {
+            const parents = new Set<string>();
+            for (const f of engine.GetIntegrationObjectFields(io.ID)) {
+                const parentID = f.RelatedIntegrationObjectID?.toUpperCase();
+                if (!parentID || UUIDsEqual(parentID, io.ID)) continue;
+                const parentName = nameById.get(parentID);
+                if (parentName) parents.add(parentName);
+            }
+            const cfgRaw = (io as unknown as { Configuration?: string | null }).Configuration;
+            if (cfgRaw) {
+                try {
+                    const cfg = JSON.parse(cfgRaw) as {
+                        parentObjectName?: string; ReferencedType?: string; parentObjectNames?: Record<string, string>;
+                    };
+                    const candidates: string[] = [];
+                    if (cfg.parentObjectName) candidates.push(cfg.parentObjectName);
+                    if (cfg.ReferencedType) candidates.push(cfg.ReferencedType);
+                    if (cfg.parentObjectNames && typeof cfg.parentObjectNames === 'object' && !Array.isArray(cfg.parentObjectNames)) {
+                        candidates.push(...Object.values(cfg.parentObjectNames).filter((v): v is string => typeof v === 'string'));
+                    }
+                    for (const c of candidates) {
+                        const match = byLowerName.get(c.toLowerCase());
+                        if (match && !UUIDsEqual(match.ID, io.ID)) parents.add(match.Name);
+                    }
+                } catch { /* non-JSON Configuration → no soft-FK parents */ }
+            }
+            deps.set(io.Name.toLowerCase(), [...parents].sort());
+        }
+        return deps;
+    }
+
     /**
      * Discovers fields on a specific external object.
      */
@@ -1266,6 +1346,13 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         @Arg("deactivateAbsent", { nullable: true, description: "Comprehensive refresh (default true): objects/fields ABSENT from this discovery are deactivated (Status='Disabled', never deleted, reversible on a later rediscovery). Pass false for a scoped/partial discovery so it never disables what it didn't probe." }) deactivateAbsent: boolean | undefined,
         @Ctx() ctx: AppContext
     ): Promise<RefreshConnectorSchemaOutput> {
+        // sync lock: a metadata refresh rewrites the IO/IOF rows a sync reads.
+        if (!IntegrationEngine.AcquireMaintenanceLock(companyIntegrationID, 'metadata refresh')) {
+            return {
+                Success: false, RunID: 'not-started',
+                Message: 'Refresh not started: a sync or another maintenance operation is currently running for this connection. Retry after it completes.',
+            };
+        }
         try {
             const user = this.getAuthenticatedUser(ctx);
             const provider = GetReadWriteProvider(ctx.providers) as unknown as IMetadataProvider;
@@ -1327,6 +1414,8 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 Message: `Error: ${this.formatError(e)}`,
                 RunID: 'error',
             };
+        } finally {
+            IntegrationEngine.ReleaseMaintenanceLock(companyIntegrationID);
         }
     }
 
@@ -3162,7 +3251,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 Message: `Batch complete: ${batchResult.SuccessCount} succeeded, ${batchResult.FailureCount} failed`,
                 Items: itemResults,
                 Steps: batchResult.Results[0]?.Steps.map((s: RSUPipelineStep) => ({
-                    Name: s.Name, Status: s.Status, DurationMs: s.DurationMs, Message: s.Message,
+                    Name: s.Name, Status: s.Status, DurationMs: s.DurationMs, Message: s.Message, StepIndex: s.StepIndex, StepTotal: s.StepTotal,
                 })),
                 GitCommitSuccess: batchResult.Results[0]?.GitCommitSuccess,
                 APIRestarted: batchResult.Results[0]?.APIRestarted,
@@ -3313,6 +3402,20 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 sourceObjectFields[resolvedNames[i]] = resolved.sourceObjects[i].Fields ?? null;
             }
 
+            // the sync after a connector's FIRST apply defaults to FULL ("always full
+            // generally as the default"); later applies default incremental. "First" = no entity
+            // map existed for this connection before this apply. Explicit input always wins.
+            const priorMapsResult = await new RunView().RunView<{ ID: string }>({
+                EntityName: 'MJ: Company Integration Entity Maps',
+                ExtraFilter: `CompanyIntegrationID='${input.CompanyIntegrationID.replace(/'/g, "''")}'`,
+                Fields: ['ID'],
+                MaxRows: 1,
+                ResultType: 'simple',
+                BypassCache: true,
+            }, user);
+            const hadPriorMaps = priorMapsResult.Success && priorMapsResult.Results.length > 0;
+            const effectiveFullSync = input.FullSync ?? !hadPriorMaps;
+
             const pendingPayload = {
                 CompanyIntegrationID: input.CompanyIntegrationID,
                 SourceObjectNames: resolvedNames,
@@ -3321,21 +3424,24 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 CronExpression: input.CronExpression,
                 ScheduleTimezone: input.ScheduleTimezone,
                 StartSync: input.StartSync,
-                FullSync: input.FullSync ?? false,
+                FullSync: effectiveFullSync,
                 SyncScope: input.SyncScope ?? 'created',
+                UnselectedAction: (input.UnselectedAction ?? 'disable') as 'disable' | 'ignore',
                 CreatedAt: new Date().toISOString(),
             };
             rsuInput.PostRestartFiles = [
                 { Path: pendingFilePath, Content: JSON.stringify(pendingPayload, null, 2) }
             ];
 
-            // Step 5: Run pipeline (restart kills process at the end)
+            // Step 5: Run pipeline (restart kills process at the end).
+            // Retrying variant — an install should not die because the migration hit a dropped
+            // connection or a transient lock; it only replays while nothing has been applied.
             const rsm = RuntimeSchemaManager.Instance;
-            const batchResult = await rsm.RunPipelineBatch([rsuInput]);
+            const batchResult = await rsm.RunPipelineBatchWithRetry([rsuInput]);
 
             const migrationSucceeded = batchResult.SuccessCount > 0;
             const pipelineSteps = batchResult.Results[0]?.Steps.map((s: RSUPipelineStep) => ({
-                Name: s.Name, Status: s.Status, DurationMs: s.DurationMs, Message: s.Message,
+                Name: s.Name, Status: s.Status, DurationMs: s.DurationMs, Message: s.Message, StepIndex: s.StepIndex, StepTotal: s.StepTotal,
             }));
 
             // If pipeline failed, clean up pending file and return error
@@ -3358,6 +3464,17 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                     input.CompanyIntegrationID, objects, connector, companyIntegration, schemaName, user, provider,
                     input.DefaultSyncDirection ?? 'Pull'
                 );
+                // Remove-as-disable: objects absent from THIS selection get their entity
+                // map + field maps disabled (data kept; re-selection re-enables). 'ignore' opts out
+                // for additive/subset applies (e.g. test-harness per-scope runs).
+                if ((input.UnselectedAction ?? 'disable') !== 'ignore') {
+                    const disabledObjects = await DisableUnselectedEntityMaps(
+                        input.CompanyIntegrationID, resolvedNames, user, provider
+                    );
+                    if (disabledObjects.length > 0) {
+                        console.log(`[IntegrationApplyAll] Disabled ${disabledObjects.length} unselected entity map(s): ${disabledObjects.join(', ')}`);
+                    }
+                }
                 const createdMapIDs = entityMapsCreated.map(em => em.EntityMapID).filter(Boolean);
                 const scopedMapIDs = input.SyncScope === 'all' ? undefined : createdMapIDs;
                 // Skip sync when SyncScope='created' but 0 new maps were
@@ -3367,7 +3484,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 const shouldStartSync = input.StartSync !== false &&
                     (input.SyncScope === 'all' || createdMapIDs.length > 0);
                 const syncRunID = shouldStartSync
-                    ? await this.startSyncAfterApply(input.CompanyIntegrationID, user, scopedMapIDs, input.FullSync)
+                    ? await this.startSyncAfterApply(input.CompanyIntegrationID, user, scopedMapIDs, effectiveFullSync)
                     : null;
 
                 // Create schedule if requested
@@ -3531,6 +3648,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         }
 
         let em: MJCompanyIntegrationEntityMapEntity;
+        const wasDisabled = existing ? existing.Status !== 'Active' || existing.SyncEnabled === false : false;
         if (existing) {
             em = existing; // reuse — keeps the map stable across re-applies
         } else {
@@ -3550,6 +3668,13 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             return null;
         }
 
+        // Re-add re-enables: a previously-removed (disabled) object that is re-selected
+        // comes back WITH its field maps — the create-field-maps idempotency below skips existing
+        // rows, so without this they would stay Disabled forever.
+        if (wasDisabled) {
+            await ReenableFieldMapsForEntityMap(em.ID, user, md);
+        }
+
         // Discover fields from the source and create 1:1 field maps
         const fieldMapCount = await this.createFieldMapsForEntityMap(
             em.ID, obj.SourceObjectName, connector, companyIntegration, user, md
@@ -3601,7 +3726,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 fm.EntityMapID = entityMapID;
                 fm.SourceFieldName = field.Name;
                 fm.DestinationFieldName = field.Name.replace(/[^A-Za-z0-9_]/g, '_');
-                fm.IsKeyField = field.IsUniqueKey;
+                fm.IsKeyField = field.IsPrimaryKey ?? false; // key on the record's PRIMARY key (identity), consistent with processRSUPendingWork + U1 (PK ≠ unique)
                 fm.IsRequired = field.IsRequired;
                 fm.Direction = 'SourceToDest';
                 fm.Status = 'Active';
@@ -3854,6 +3979,14 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 return { Success: false, Message: 'Connector is deactivated (IsActive=false); sync not started' };
             }
 
+            // sync lock: refuse while a metadata refresh / schema evolution / RSU pipeline
+            // holds the maintenance lock for this connection (the engine also refuses — this gives
+            // the client an immediate, unambiguous answer).
+            const maintenance = IntegrationEngine.GetMaintenanceLock(companyIntegrationID);
+            if (maintenance) {
+                return { Success: false, Message: `Sync not started: ${maintenance.Reason} is in progress for this connection (since ${maintenance.AcquiredAt.toISOString()}). Retry after it completes.` };
+            }
+
             const syncOptions: { FullSync?: boolean; EntityMapIDs?: string[]; SyncDirection?: 'Pull' | 'Push' | 'Bidirectional' } = {};
             if (fullSync) syncOptions.FullSync = true;
             if (entityMapIDs?.length) syncOptions.EntityMapIDs = entityMapIDs;
@@ -3910,10 +4043,21 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             // null RunID: an optimistic, untrackable result that read as "started" when nothing
             // could be followed. Poll briefly for ANY run for this connector stamped at/after the
             // fire instant; that catches both the still-running and the already-finished cases.
+            //
+            // The row itself isn't created until executeSyncInternal reaches CreateRunRecord,
+            // which runs AFTER LoadRunConfiguration (loads every entity map + field map + builds
+            // the connector instance) — for a large connector (HubSpot, 168 entity maps) that
+            // synchronous setup alone can take several seconds, so a short fixed poll window can
+            // elapse before the row exists at all, not because the sync failed. Live-verified: a
+            // HubSpot StartSync call reported "no run created" while the run (confirmed present in
+            // the DB moments later) was genuinely syncing. Keep the fast path fast (first 2s still
+            // polled every 200ms) but extend the tail so large connectors get a fair window before
+            // we honestly report failure.
             const firedAtFilter = firedAt.toISOString();
             let run: { ID: string; Status: string } | null = null;
-            for (let attempt = 0; attempt < 15 && !run; attempt++) {
-                await new Promise(resolve => setTimeout(resolve, 200));
+            const pollIntervalsMs = [...Array(10).fill(200), ...Array(20).fill(1000)];
+            for (let attempt = 0; attempt < pollIntervalsMs.length && !run; attempt++) {
+                await new Promise(resolve => setTimeout(resolve, pollIntervalsMs[attempt]));
                 const rv = new RunView();
                 const runResult = await rv.RunView<MJCompanyIntegrationRunEntity>({
                     EntityName: 'MJ: Company Integration Runs',
@@ -4087,8 +4231,18 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             }
             const jobTypeID = jobTypeResult.Results[0].ID;
 
+            // U7 — SERVER-SIDE uniqueness per (CompanyIntegration, job kind/driver): a client
+            // name-regex is not a guarantee, and duplicate jobs mean duplicate sync/rediscovery
+            // runs. When an active/paused job of this kind already targets this connection,
+            // UPDATE it in place instead of creating a twin.
             const job = await md.GetEntityObject<MJScheduledJobEntity>('MJ: Scheduled Jobs', user);
-            job.NewRecord();
+            const existingJob = await this.findExistingScheduledJob(jobTypeID, input.CompanyIntegrationID, user);
+            let updatedInPlace = false;
+            if (existingJob) {
+                const loaded = await job.InnerLoad(CompositeKey.FromID(existingJob.ID));
+                if (loaded) updatedInPlace = true;
+            }
+            if (!updatedInPlace) job.NewRecord();
             job.JobTypeID = jobTypeID;
             job.Name = input.Name;
             if (input.Description) job.Description = input.Description;
@@ -4120,13 +4274,50 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 await ci.Save();
             }
 
-            return { Success: true, Message: 'Schedule created', ScheduledJobID: job.ID };
+            return {
+                Success: true,
+                Message: updatedInPlace
+                    ? 'Schedule updated — an active schedule of this kind already existed for this connection (server-side uniqueness)'
+                    : 'Schedule created',
+                ScheduledJobID: job.ID,
+            };
         } catch (e) {
             LogError(`IntegrationCreateSchedule error: ${e}`);
             return { Success: false, Message: this.formatError(e) };
         }
     }
 
+    /**
+     * U7 — server-side schedule uniqueness: finds an existing (non-deleted) scheduled job of the
+     * given job type that targets the given CompanyIntegration (its Configuration JSON carries
+     * CompanyIntegrationID). Matched case-insensitively across UUID casings (SQL Server upper /
+     * PostgreSQL lower). Returns the newest match or null.
+     */
+    private async findExistingScheduledJob(
+        jobTypeID: string,
+        companyIntegrationID: string,
+        user: UserInfo
+    ): Promise<{ ID: string } | null> {
+        const esc = (s: string) => s.replace(/'/g, "''");
+        const lower = esc(companyIntegrationID.toLowerCase());
+        const upper = esc(companyIntegrationID.toUpperCase());
+        const result = await new RunView().RunView<{ ID: string }>({
+            EntityName: 'MJ: Scheduled Jobs',
+            ExtraFilter:
+                `JobTypeID='${esc(jobTypeID)}' AND Status IN ('Active','Paused') AND ` +
+                // Structured "CompanyIntegrationID":"<uuid>" predicate (same form as IntegrationListSchedules),
+                // NOT a bare-UUID substring — so this CI's id can't false-match the same UUID appearing in
+                // another config key (e.g. RelatedCompanyIntegrationID) and repurpose an unrelated job.
+                `(Configuration LIKE '%"CompanyIntegrationID":"${lower}"%' OR Configuration LIKE '%"CompanyIntegrationID":"${upper}"%')`,
+            OrderBy: '__mj_CreatedAt DESC',
+            Fields: ['ID'],
+            MaxRows: 1,
+            ResultType: 'simple',
+            BypassCache: true, // uniqueness must see committed jobs, not a stale filtered cache
+        }, user);
+        return result.Success && result.Results.length > 0 ? result.Results[0] : null;
+    }
+
     @Mutation(() => MutationResultOutput)
     async IntegrationUpdateSchedule(
         @Arg("scheduledJobID") scheduledJobID: string,
@@ -4491,7 +4682,10 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                     OperationType: 'rsu',
                     IsRunning: true,
                     RSURunning: true,
-                    RSUStep: rsuStatus.LastRunResult ?? 'running',
+                    // U11 — determinate "step N of M: <name>" instead of an indeterminate spinner
+                    RSUStep: rsuStatus.CurrentStepName ?? rsuStatus.LastRunResult ?? 'running',
+                    RSUStepIndex: rsuStatus.CurrentStepIndex ?? undefined,
+                    RSUStepTotal: rsuStatus.StepTotal ?? undefined,
                     StartedAt: rsuStatus.LastRunAt?.toISOString(),
                     ElapsedMs: rsuStatus.LastRunAt ? Date.now() - rsuStatus.LastRunAt.getTime() : undefined,
                 };
@@ -5055,6 +5249,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                         SyncScope: input.SyncScope ?? 'created',
                         SyncDirection: input.SyncDirection,
                         ScheduleSyncDirection: input.ScheduleSyncDirection,
+                        UnselectedAction: (input.UnselectedAction ?? 'disable') as 'disable' | 'ignore',
                         CreatedAt: new Date().toISOString(),
                     };
                     rsuInput.PostRestartFiles = [
@@ -5113,10 +5308,13 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 };
             }
 
-            // Phase 2: Run all successful RSU inputs through one pipeline batch
+            // Phase 2: Run all successful RSU inputs through one pipeline batch.
+            // Retrying variant — a multi-connector install is the longest-running thing a user
+            // does here, and it should not be lost to one transient step. Replays only while
+            // nothing has been applied, so a partially-committed batch is never re-executed.
             const pipelineInputs = successfulBuilds.map(b => b.rsuInput);
             const rsm = RuntimeSchemaManager.Instance;
-            const batchResult = await rsm.RunPipelineBatch(pipelineInputs);
+            const batchResult = await rsm.RunPipelineBatchWithRetry(pipelineInputs);
 
             // Phase 3: Post-pipeline — create entity maps, field maps, schedules for each success
             for (let i = 0; i < successfulBuilds.length; i++) {
@@ -5190,7 +5388,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             }
 
             const pipelineSteps = batchResult.Results[0]?.Steps.map((s: RSUPipelineStep) => ({
-                Name: s.Name, Status: s.Status, DurationMs: s.DurationMs, Message: s.Message,
+                Name: s.Name, Status: s.Status, DurationMs: s.DurationMs, Message: s.Message, StepIndex: s.StepIndex, StepTotal: s.StepTotal,
             }));
 
             const successCount = connectorResults.filter(r => r.Success).length;
@@ -5242,8 +5440,12 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             }
             const jobTypeID = jobTypeResult.Results[0].ID;
 
+            // U7 — same server-side uniqueness as IntegrationCreateSchedule: update the existing
+            // sync job for this connection in place rather than creating a duplicate.
             const job = await md.GetEntityObject<MJScheduledJobEntity>('MJ: Scheduled Jobs', user);
-            job.NewRecord();
+            const existingJob = await this.findExistingScheduledJob(jobTypeID, companyIntegrationID, user);
+            const loadedExisting = existingJob ? await job.InnerLoad(CompositeKey.FromID(existingJob.ID)) : false;
+            if (!loadedExisting) job.NewRecord();
             job.JobTypeID = jobTypeID;
             job.Name = `${integrationName} Sync`;
             job.CronExpression = cronExpression;
@@ -5492,8 +5694,19 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         @Arg("platform", { defaultValue: "sqlserver" }) platform: string,
         @Arg("skipGitCommit", { defaultValue: false }) skipGitCommit: boolean,
         @Arg("skipRestart", { defaultValue: false }) skipRestart: boolean,
+        @Arg("autoEnableNewObjects", { defaultValue: false, description: 'newly-appeared objects get their entity maps created DISABLED by default (the user enables them after the refresh). Pass true to auto-enable them instead.' }) autoEnableNewObjects: boolean,
+        @Arg("deactivateAbsent", { nullable: true, description: 'Deactivate IO/IOF absent from this re-discovery (default true — comprehensive refresh; gated on the connector\'s authoritative-discovery getter).' }) deactivateAbsent: boolean | undefined,
+        @Arg("cascadeRemoveDependents", { defaultValue: false, description: 'When a removed object has still-active dependents (DAG parent edges), also disable the transitive dependent closure ("force remove those too"). Default false: dependents stay active and each broken edge is surfaced as a warning.' }) cascadeRemoveDependents: boolean,
         @Ctx() ctx: AppContext
     ): Promise<SchemaEvolutionOutput> {
+        // sync lock: schema evolution rewrites the metadata, field maps and DDL a sync
+        // reads — data syncs (manual + scheduled) must not run while this is in flight.
+        if (!IntegrationEngine.AcquireMaintenanceLock(companyIntegrationID, 'schema evolution')) {
+            return {
+                Success: false, HasChanges: false,
+                Message: 'Schema evolution not started: a sync or another maintenance operation is currently running for this connection. Retry after it completes.',
+            };
+        }
         try {
             const user = this.getAuthenticatedUser(ctx);
             const validatedPlatform = this.validatePlatform(platform);
@@ -5502,25 +5715,92 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             const schemaName = this.deriveSchemaName(companyIntegration.Integration);
             const rv = new RunView();
 
-            // Step 1: Get existing entity maps for this CompanyIntegration
-            const entityMapsResult = await rv.RunView<MJCompanyIntegrationEntityMapEntity>({
+            // ── Phase 1 — FULL RE-RESOLUTION (: "the exact same algorithm as before") ──
+            // Re-run the creation pipeline: introspect (static + runtime discovery), persist with
+            // the external-wins-when-present overlay, deactivate absent (gated on authoritative
+            // discovery), PK-classify the residual. The diff below is computed against the
+            // RESULT of this resolution, exactly as the first-time flow computed its state.
+            const pipeline = new IntegrationConnectorCreationPipeline();
+            const refresh = await pipeline.Run({
+                Connector: connector,
+                CompanyIntegration: companyIntegration,
+                ContextUser: user,
+                Provider: md,
+                TriggerType: 'Manual',
+                ConsoleMirror: true,
+                DeactivateAbsent: deactivateAbsent ?? this.readConfigBool(companyIntegration.Configuration, 'deactivateAbsent') ?? true,
+            } as unknown as Parameters<typeof pipeline.Run>[0]);
+            if (!refresh.Success) {
+                return { Success: false, HasChanges: false, Message: `Schema evolution aborted — metadata re-resolution failed: ${refresh.FailureMessage ?? 'unknown error'}` };
+            }
+            await md.Refresh();
+            await IntegrationEngine.Instance.Config(true, user, md);
+
+            // ── Phase 2 — DIFF: resolved-Active objects vs prior entity maps ──
+            const activeIOs = IntegrationEngineBase.Instance.GetActiveIntegrationObjects(companyIntegration.IntegrationID);
+            const activeNamesLower = new Set(activeIOs.map(o => o.Name.toLowerCase()));
+            const priorMapsResult = await rv.RunView<MJCompanyIntegrationEntityMapEntity>({
                 EntityName: 'MJ: Company Integration Entity Maps',
                 ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}'`,
-                ResultType: 'simple',
-                Fields: ['ID', 'ExternalObjectName', 'EntityID']
+                ResultType: 'entity_object',
+                BypassCache: true, // the diff must read committed state
             }, user);
+            const priorMaps = priorMapsResult.Success ? priorMapsResult.Results : [];
+            const priorByLowerName = new Map(priorMaps.map(m => [(m.ExternalObjectName ?? '').toLowerCase(), m] as const));
+
+            const newObjects = activeIOs.filter(io => !priorByLowerName.has(io.Name.toLowerCase())).map(io => io.Name);
+            const removedObjects = priorMaps
+                .filter(m => m.ExternalObjectName && !activeNamesLower.has(m.ExternalObjectName.toLowerCase()))
+                .map(m => m.ExternalObjectName as string);
+            const continuingMaps = priorMaps.filter(m => m.ExternalObjectName && activeNamesLower.has(m.ExternalObjectName.toLowerCase()));
+
+            // ── Phase 3 — REMOVED objects: disable EM + EFMs (data + tables KEPT) ──
+            // Uses the same remove-as-disable helper as ApplyAll: everything not in the resolved
+            // set is disabled; a later re-appearance re-enables (reactivate-on-rediscover).
+            // DAG handling (rsuplan "adding and removing tables"): dependents of a removed object
+            // are NOT auto-removed by default — each broken parent edge is surfaced as a warning so
+            // a consumer can prompt the user. With cascadeRemoveDependents=true (the "force remove
+            // those too" affordance), the TRANSITIVE closure of still-active dependents is disabled
+            // alongside the removed objects (still remove-as-disable, reversible). Edges come from
+            // the same dependency map the sync engine layers by.
+            const depMap = removedObjects.length > 0
+                ? this.computeObjectDependencyNames(companyIntegration.IntegrationID)
+                : new Map<string, string[]>();
+            const cascadeRemoved: string[] = cascadeRemoveDependents && removedObjects.length > 0
+                ? ComputeCascadeRemovalSet(activeIOs.map(io => io.Name), removedObjects, depMap)
+                : [];
+            const cascadeLower = new Set(cascadeRemoved.map(n => n.toLowerCase()));
+            const keepNames = activeIOs.map(io => io.Name).filter(n => !cascadeLower.has(n.toLowerCase()));
+            const dagRemovalWarnings: string[] = removedObjects.length > 0 && !cascadeRemoveDependents
+                ? ComputeRemovedDependencyWarnings(keepNames, removedObjects, depMap)
+                : [];
+            for (const w of dagRemovalWarnings) LogStatus(`[SchemaEvolution][DAG] ${w}`);
+            if (cascadeRemoved.length > 0) LogStatus(`[SchemaEvolution][DAG] cascade-removing ${cascadeRemoved.length} dependent object(s): ${cascadeRemoved.join(', ')}`);
+            if (removedObjects.length > 0 || cascadeRemoved.length > 0) {
+                await DisableUnselectedEntityMaps(companyIntegrationID, keepNames, user, md);
+            }
 
-            if (!entityMapsResult.Success || entityMapsResult.Results.length === 0) {
+            // ── Phase 4 — schema build over continuing + new objects (minus cascade-removed) ──
+            const evolveNames = [
+                ...continuingMaps.map(m => m.ExternalObjectName as string).filter(n => !cascadeLower.has(n.toLowerCase())),
+                ...newObjects,
+            ];
+            if (evolveNames.length === 0) {
+                const removedTotal = removedObjects.length + cascadeRemoved.length;
                 return {
-                    Success: false, Message: 'No entity maps found — nothing to evolve',
-                    HasChanges: false,
+                    Success: true, HasChanges: removedTotal > 0,
+                    Message: removedTotal > 0
+                        ? `No objects to evolve — ${removedTotal} removed object(s) disabled${cascadeRemoved.length > 0 ? ` (${cascadeRemoved.length} via cascade)` : ''}`
+                        : 'No entity maps and no resolved objects — nothing to evolve',
+                    RemovedObjects: removedObjects.length > 0 ? removedObjects : undefined,
+                    CascadeRemovedObjects: cascadeRemoved.length > 0 ? cascadeRemoved : undefined,
+                    Warnings: dagRemovalWarnings.length > 0 ? dagRemovalWarnings : undefined,
                 };
             }
+            const objects = this.buildObjectInputsFromNames(evolveNames, schemaName);
 
-            const sourceObjectNames = entityMapsResult.Results.map(em => em.ExternalObjectName);
-            const objects = this.buildObjectInputsFromNames(sourceObjectNames, schemaName);
-
-            // Step 2: Build ExistingTables from Metadata.Entities matching the schema
+            // Build ExistingTables from Metadata.Entities matching the schema (continuing objects
+            // have a table; new objects don't — SchemaBuilder CREATEs those, ALTERs the rest)
             const existingTables: ExistingTableInfo[] = [];
             for (const obj of objects) {
                 const entityInfo = md.Entities.find(
@@ -5543,14 +5823,13 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 }
             }
 
-            // Step 3: Introspect current schema from connector
-            const introspect = connector.IntrospectSchema.bind(connector) as
-                (ci: unknown, u: unknown) => Promise<SourceSchemaInfo>;
-            const sourceSchema = await introspect(companyIntegration, user);
+            // Source of truth for the evolution = the PERSISTED post-resolution rows (the refresh in
+            // Phase 1 just wrote them via the overlay) — no duplicate vendor introspect.
+            const sourceSchema = this.buildSourceSchemaFromPersistedRows(companyIntegration.IntegrationID);
 
             // Normalize names to match source schema casing
             const nameMap = new Map(sourceSchema.Objects.map(o => [o.ExternalName.toLowerCase(), o.ExternalName]));
-            const normalizedNames = sourceObjectNames.map(n => nameMap.get(n.toLowerCase()) ?? n);
+            const normalizedNames = evolveNames.map(n => nameMap.get(n.toLowerCase()) ?? n);
             // Also normalize the objects array SourceObjectName
             for (const obj of objects) {
                 const exact = nameMap.get(obj.SourceObjectName.toLowerCase());
@@ -5562,7 +5841,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 Objects: sourceSchema.Objects.filter(o => requestedNames.has(o.ExternalName))
             };
 
-            // Step 4: Build target configs and SchemaBuilder input with ExistingTables
+            // Build target configs and SchemaBuilder input with ExistingTables
             const targetConfigs = this.buildTargetConfigs(objects, filteredSchema, validatedPlatform, connector);
 
             const schemaInput: SchemaBuilderInput = {
@@ -5578,7 +5857,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 EntitySettingsForTargets: {}
             };
 
-            // Step 5: Build schema — SchemaBuilder handles evolution (ALTER TABLE) internally
+            // ── Phase 5 — build schema (SchemaBuilder CREATEs new tables, ALTERs changed ones) ──
             const builder = new SchemaBuilder();
             const schemaOutput = builder.BuildSchema(schemaInput);
 
@@ -5591,19 +5870,10 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 };
             }
 
-            // Step 6: Check if any migration SQL was generated
-            if (schemaOutput.MigrationFiles.length === 0) {
-                return {
-                    Success: true,
-                    Message: 'No schema changes detected',
-                    HasChanges: false,
-                    Warnings: schemaOutput.Warnings.length > 0 ? schemaOutput.Warnings : undefined,
-                };
-            }
-
-            // Step 7: Count added/modified columns by re-diffing
+            // Per-object DDL diff for CHANGED detection (drives U10 watermark resets + reporting)
             let addedColumns = 0;
             let modifiedColumns = 0;
+            const changedObjects: string[] = [];
             const evolution = new SchemaEvolution();
             for (const existing of existingTables) {
                 const config = targetConfigs.find(
@@ -5616,39 +5886,228 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 const diff = evolution.DiffSchema(sourceObj, config, existing, validatedPlatform);
                 addedColumns += diff.AddedColumns.length;
                 modifiedColumns += diff.ModifiedColumns.length;
+                if (diff.AddedColumns.length > 0 || diff.ModifiedColumns.length > 0) {
+                    changedObjects.push(config.SourceObjectName);
+                }
             }
 
-            // Step 8: Run RSU pipeline
-            const rsuInput = builder.BuildRSUInput(schemaOutput, schemaInput, {
-                SkipGitCommit: skipGitCommit,
-                SkipRestart: skipRestart,
-            });
+            // ── Phase 6 — reconcile CONTINUING maps' field maps to the resolution ──
+            // Added source fields get field maps (Active on enabled maps, Disabled on disabled
+            // maps); field maps whose source field vanished from the resolution are disabled.
+            // Field maps carry no entity FK, so this is restart-safe done here, pre-RSU.
+            const iosByLowerName = new Map(activeIOs.map(io => [io.Name.toLowerCase(), io] as const));
+            for (const em of continuingMaps) {
+                const io = iosByLowerName.get((em.ExternalObjectName ?? '').toLowerCase());
+                if (!io) continue;
+                const fmChanges = await this.reconcileFieldMapsForEntityMap(em, io.ID, user, md);
+                if ((fmChanges.Added > 0 || fmChanges.Disabled > 0) && em.ExternalObjectName && !changedObjects.includes(em.ExternalObjectName)) {
+                    changedObjects.push(em.ExternalObjectName);
+                }
+            }
 
-            const rsm = RuntimeSchemaManager.Instance;
-            const batchResult = await rsm.RunPipelineBatch([rsuInput]);
+            // ── Phase 7 — U10: reset the Pull watermark of every CHANGED object so the next sync
+            // full-fetches + backfills the schema change (content-hash bounds the rewrite cost).
+            const changedLower = new Set(changedObjects.map(n => n.toLowerCase()));
+            const changedMapIDs = continuingMaps
+                .filter(m => m.ExternalObjectName && changedLower.has(m.ExternalObjectName.toLowerCase()))
+                .map(m => m.ID);
+            const resetMapIDs = await ResetPullWatermarks(changedMapIDs, user, md);
+            const watermarksReset = continuingMaps
+                .filter(m => resetMapIDs.some(id => UUIDsEqual(id, m.ID)))
+                .map(m => m.ExternalObjectName as string);
+
+            const hasDDLChanges = schemaOutput.MigrationFiles.length > 0;
+            const hasChanges = hasDDLChanges || newObjects.length > 0 || removedObjects.length > 0 || changedObjects.length > 0;
+            if (!hasChanges) {
+                return {
+                    Success: true,
+                    Message: 'No schema changes detected',
+                    HasChanges: false,
+                    Warnings: schemaOutput.Warnings.length > 0 ? schemaOutput.Warnings : undefined,
+                };
+            }
 
-            const pipelineResult = batchResult.Results[0];
-            const pipelineSteps = pipelineResult?.Steps.map((s: RSUPipelineStep) => ({
-                Name: s.Name, Status: s.Status, DurationMs: s.DurationMs, Message: s.Message,
-            }));
+            // ── Phase 8 — run RSU when DDL changed (migrate → codegen → commit → compile → restart) ──
+            let pipelineSteps: RSUStepOutput[] | undefined;
+            let gitCommitSuccess: boolean | undefined;
+            let apiRestarted: boolean | undefined;
+            if (hasDDLChanges) {
+                const rsuInput = builder.BuildRSUInput(schemaOutput, schemaInput, {
+                    SkipGitCommit: skipGitCommit,
+                    SkipRestart: skipRestart,
+                    // Schema evolution runs a FULL re-resolution of this connector's schema, so its
+                    // soft-constraint payload is the complete current truth — authorize the writer to
+                    // prune additionalSchemaInfo entries for tables that vanished from the resolution
+                    // (rsuplan: "adds new ones, removes old ones that no longer exist"). Subset
+                    // ApplyAll builds deliberately do NOT set this.
+                    AdditionalSchemaInfoAuthoritative: true,
+                });
+
+                // NEW objects need entities (post-codegen/restart) before their entity maps can be
+                // created — hand them to the post-restart consumer via the SAME pending-work channel
+                // ApplyAll uses. default: created DISABLED (user enables after the refresh);
+                // autoEnableNewObjects opts in. Removal was already handled in Phase 3 → 'ignore'.
+                if (newObjects.length > 0 && !skipRestart) {
+                    const { join } = await import('node:path');
+                    const rsuWorkDir = process.env.RSU_WORK_DIR || process.cwd();
+                    const pendingFilePath = join(rsuWorkDir, '.rsu_pending', `${Date.now()}-evolution.json`);
+                    rsuInput.PostRestartFiles = [
+                        ...(rsuInput.PostRestartFiles ?? []),
+                        {
+                            Path: pendingFilePath,
+                            Content: JSON.stringify({
+                                CompanyIntegrationID: companyIntegrationID,
+                                SourceObjectNames: newObjects,
+                                SourceObjectFields: Object.fromEntries(newObjects.map(n => [n, null])),
+                                SchemaName: schemaName,
+                                StartSync: false,
+                                SyncScope: 'created',
+                                UnselectedAction: 'ignore',
+                                CreateDisabled: !autoEnableNewObjects,
+                                CreatedAt: new Date().toISOString(),
+                            }, null, 2),
+                        },
+                    ];
+                }
+
+                const rsm = RuntimeSchemaManager.Instance;
+                const batchResult = await rsm.RunPipelineBatch([rsuInput]);
+                const pipelineResult = batchResult.Results[0];
+                pipelineSteps = pipelineResult?.Steps.map((s: RSUPipelineStep) => ({
+                    Name: s.Name, Status: s.Status, DurationMs: s.DurationMs, Message: s.Message, StepIndex: s.StepIndex, StepTotal: s.StepTotal,
+                }));
+                gitCommitSuccess = pipelineResult?.GitCommitSuccess;
+                apiRestarted = pipelineResult?.APIRestarted;
+                if (!pipelineResult?.Success) {
+                    return {
+                        Success: false,
+                        Message: `Pipeline failed: ${pipelineResult?.ErrorMessage ?? 'unknown error'}`,
+                        HasChanges: true,
+                        AddedColumns: addedColumns,
+                        ModifiedColumns: modifiedColumns,
+                        Steps: pipelineSteps,
+                        NewObjects: newObjects.length > 0 ? newObjects : undefined,
+                        RemovedObjects: removedObjects.length > 0 ? removedObjects : undefined,
+                        ChangedObjects: changedObjects.length > 0 ? changedObjects : undefined,
+                        Warnings: schemaOutput.Warnings.length > 0 ? schemaOutput.Warnings : undefined,
+                    };
+                }
+
+                // skipRestart=true: entities exist after provider.Refresh() — create the new
+                // objects' entity maps inline, then apply the disabled-by-default rule.
+                if (newObjects.length > 0 && skipRestart) {
+                    await md.Refresh();
+                    const newObjectInputs = objects.filter(o => newObjects.some(n => n.toLowerCase() === o.SourceObjectName.toLowerCase()));
+                    const created = await this.createEntityAndFieldMaps(
+                        companyIntegrationID, newObjectInputs, connector, companyIntegration, schemaName, user, md, 'Pull'
+                    );
+                    if (!autoEnableNewObjects) {
+                        for (const c of created) {
+                            if (c.EntityMapID) await SetEntityMapEnabled(c.EntityMapID, false, user, md);
+                        }
+                    }
+                }
+            }
+
+            const summary = [
+                newObjects.length > 0 ? `${newObjects.length} new object(s) (${autoEnableNewObjects ? 'enabled' : 'created disabled'})` : null,
+                removedObjects.length > 0 ? `${removedObjects.length} removed object(s) disabled` : null,
+                changedObjects.length > 0 ? `${changedObjects.length} changed object(s), ${addedColumns} column(s) added, ${modifiedColumns} modified` : null,
+                watermarksReset.length > 0 ? `${watermarksReset.length} watermark(s) reset for backfill` : null,
+            ].filter(Boolean).join('; ');
 
             return {
-                Success: pipelineResult?.Success ?? false,
-                Message: pipelineResult?.Success
-                    ? `Schema evolution applied — ${addedColumns} column(s) added, ${modifiedColumns} column(s) modified`
-                    : `Pipeline failed: ${pipelineResult?.ErrorMessage ?? 'unknown error'}`,
+                Success: true,
+                Message: `Schema evolution applied — ${summary || 'no per-object deltas'}`,
                 HasChanges: true,
                 AddedColumns: addedColumns,
                 ModifiedColumns: modifiedColumns,
                 Steps: pipelineSteps,
-                GitCommitSuccess: pipelineResult?.GitCommitSuccess,
-                APIRestarted: pipelineResult?.APIRestarted,
-                Warnings: schemaOutput.Warnings.length > 0 ? schemaOutput.Warnings : undefined,
+                GitCommitSuccess: gitCommitSuccess,
+                APIRestarted: apiRestarted,
+                NewObjects: newObjects.length > 0 ? newObjects : undefined,
+                RemovedObjects: removedObjects.length > 0 ? removedObjects : undefined,
+                CascadeRemovedObjects: cascadeRemoved.length > 0 ? cascadeRemoved : undefined,
+                ChangedObjects: changedObjects.length > 0 ? changedObjects : undefined,
+                WatermarksReset: watermarksReset.length > 0 ? watermarksReset : undefined,
+                Warnings: (dagRemovalWarnings.length + schemaOutput.Warnings.length) > 0
+                    ? [...dagRemovalWarnings, ...schemaOutput.Warnings]
+                    : undefined,
             };
         } catch (e) {
             LogError(`IntegrationSchemaEvolution error: ${e}`);
             return { Success: false, Message: this.formatError(e), HasChanges: false };
+        } finally {
+            IntegrationEngine.ReleaseMaintenanceLock(companyIntegrationID);
+        }
+    }
+
+    /**
+     * Refresh diff (continuing objects): reconciles one entity map's field maps to the
+     * post-resolution IOF set — source fields WITHOUT a field map get one created (Active when
+     * the map is enabled, Disabled when it isn't), and field maps whose source field is no
+     * longer Active in the resolution are DISABLED (never deleted — a later re-appearance
+     * re-enables via the same reconciliation, since a disabled FM whose field returns flips
+     * back with SetFieldMapsStatus on re-enable or is simply left disabled until then).
+     */
+    private async reconcileFieldMapsForEntityMap(
+        em: MJCompanyIntegrationEntityMapEntity,
+        integrationObjectID: string,
+        user: UserInfo,
+        md: IMetadataProvider
+    ): Promise<{ Added: number; Disabled: number }> {
+        const result = { Added: 0, Disabled: 0 };
+        const activeFields = IntegrationEngineBase.Instance
+            .GetIntegrationObjectFields(integrationObjectID)
+            .filter(f => f.Status === 'Active');
+        const activeFieldNames = new Set(activeFields.map(f => f.Name.toLowerCase()));
+        const mapEnabled = em.Status === 'Active' && em.SyncEnabled === true;
+
+        const fms = await new RunView().RunView<MJCompanyIntegrationFieldMapEntity>({
+            EntityName: 'MJ: Company Integration Field Maps',
+            ExtraFilter: `EntityMapID='${em.ID}'`,
+            ResultType: 'entity_object',
+            BypassCache: true,
+        }, user);
+        const existingByLower = new Map(
+            (fms.Success ? fms.Results : []).map(fm => [(fm.SourceFieldName ?? '').toLowerCase(), fm] as const)
+        );
+
+        // Added source fields → new field maps (state follows the map's enabled state)
+        for (const field of activeFields) {
+            const existing = existingByLower.get(field.Name.toLowerCase());
+            if (existing) {
+                // Field is back in the resolution — re-enable a previously-disabled map row.
+                if (existing.Status !== 'Active' && mapEnabled) {
+                    existing.Status = 'Active';
+                    if (await existing.Save()) result.Added++; // counted as a change for reporting
+                }
+                continue;
+            }
+            const fm = await md.GetEntityObject<MJCompanyIntegrationFieldMapEntity>('MJ: Company Integration Field Maps', user);
+            fm.NewRecord();
+            fm.EntityMapID = em.ID;
+            fm.SourceFieldName = field.Name;
+            fm.DestinationFieldName = field.Name.replace(/[^A-Za-z0-9_]/g, '_');
+            fm.IsKeyField = field.IsPrimaryKey ?? false; // key on the PRIMARY key (identity), consistent with the initial-apply path + U1 (PK ≠ unique)
+            fm.IsRequired = field.IsRequired === true;
+            fm.Direction = 'SourceToDest';
+            fm.Status = mapEnabled ? 'Active' : 'Inactive';
+            fm.Priority = 0;
+            if (await fm.Save()) result.Added++;
+            else LogError(`[IntegrationSchemaEvolution] Failed to create field map '${field.Name}' on ${em.ExternalObjectName}: ${fm.LatestResult?.CompleteMessage ?? 'unknown error'}`);
         }
+
+        // Vanished source fields → disable their field maps (data/columns kept)
+        for (const [lowerName, fm] of existingByLower) {
+            if (activeFieldNames.has(lowerName)) continue;
+            if (fm.Status === 'Inactive') continue;
+            fm.Status = 'Inactive';
+            if (await fm.Save()) result.Disabled++;
+            else LogError(`[IntegrationSchemaEvolution] Failed to disable field map '${fm.SourceFieldName}' on ${em.ExternalObjectName}: ${fm.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+        }
+
+        return result;
     }
 
     // ── WEBHOOK HELPER ──────────────────────────────────────────────────