@memberjunction/server 6.2.0-edge.5 → 6.2.0-edge.7

package/src/integration/ConnectionPause.ts

@@ -0,0 +1,222 @@
+/**
+ * ConnectionPause — what "pause this connection" has to mean, beyond a flag.
+ *
+ * `IntegrationDeactivateConnection` used to set `IsActive = false` and stop. The engine reads
+ * that flag before a sync, so syncs did stop — but nothing else did:
+ *
+ *   - The SCHEDULED jobs kept firing. A sync schedule woke on cron, was refused, and wrote a
+ *     failed run. Every cron tick, for as long as the connection stayed paused, so a paused
+ *     connector's history filled with failures it did not cause.
+ *   - The DISCOVERY schedule kept firing and was NOT refused, because nothing gated discovery
+ *     on IsActive. A paused connection went on rescanning the vendor on a timer — spending the
+ *     customer's rate budget, and rewriting the very catalog the pause was meant to freeze.
+ *   - An IN-FLIGHT sync ran to completion. "Paused" arrived only after the thing the operator
+ *     was watching had finished.
+ *
+ * So pause has to touch three things, and resume has to put back exactly what pause took —
+ * which is the part that needs state rather than a rule.
+ *
+ * WHY THE PRIOR STATE IS RECORDED RATHER THAN INFERRED. "Resume = set every schedule Active" is
+ * wrong: a connection can legitimately be paused while one of its two schedules was ALREADY off
+ * because the operator turned it off weeks ago. Inferring would silently re-enable it, and the
+ * first symptom is a sync nobody asked for. So pause records which jobs it actually moved, and
+ * resume restores only those.
+ *
+ * WHY PAUSE IS IDEMPOTENT AT THE RECORD, NOT AT THE WRITE. Deactivating an already-deactivated
+ * connection finds every job already Paused, so the set it "moved" is empty. Writing that empty
+ * set over the real record is the bug that loses the state — resume would then restore nothing
+ * and both schedules would stay off forever, with no error anywhere. {@link DecidePauseWrite}
+ * is the guard: the FIRST pause's record stands until a resume consumes it, and a later pause only
+ * ever adds to it.
+ *
+ * Everything here is a pure function over plain values. The resolver owns the reads and writes.
+ */
+
+import { UUIDSet } from '@memberjunction/global';
+
+/** The two schedule kinds a connection can own. Mirrors ScheduledJob's DriverClass split. */
+export type PausedScheduleKind = 'sync' | 'discovery';
+
+/** One schedule that PAUSE moved from Active to Paused, and that RESUME should move back. */
+export interface PausedScheduleRecord {
+    /** ScheduledJob.ID */
+    ID: string;
+    /** Recorded for the operator-facing message only; restoration keys on ID. */
+    Kind: PausedScheduleKind;
+}
+
+/** Minimal shape this module needs from a ScheduledJob row. */
+export interface ScheduleJobState {
+    ID: string;
+    Status: string | null;
+    Kind: PausedScheduleKind;
+}
+
+/** Key under CompanyIntegration.Configuration where the record lives. */
+export const PAUSED_SCHEDULES_KEY = 'pausedSchedules';
+
+/**
+ * Which schedules a pause should move, and the record to remember them by.
+ *
+ * Only jobs that are currently `Active` are moved. A job already `Paused`, or in any other
+ * status, is left exactly as found and is deliberately absent from the record — that absence
+ * is what stops resume from turning it on.
+ */
+export function DecideSchedulesToPause(jobs: ReadonlyArray<ScheduleJobState>): PausedScheduleRecord[] {
+    return jobs
+        .filter(j => j.Status === 'Active')
+        .map(j => ({ ID: j.ID, Kind: j.Kind }));
+}
+
+/**
+ * Whether this pause's record should be written, given whatever record is already stored.
+ *
+ * Returns the record to persist, or `null` to leave the stored one untouched. A second pause of
+ * an already-paused connection moves nothing, and must not overwrite the first pause's record
+ * with its own empty one — see the class comment.
+ *
+ * A pause that DOES move jobs while a record is stored adds them to it: the stored entries stay
+ * exactly as they were, and the new ones are appended by job ID. That happens when a job was
+ * created or switched back on while the connection was paused, or when a resume could not clear
+ * its record. A job paused here but left out of the record is one resume can never put back.
+ */
+export function DecidePauseWrite(
+    stored: ReadonlyArray<PausedScheduleRecord>,
+    newlyPaused: ReadonlyArray<PausedScheduleRecord>
+): PausedScheduleRecord[] | null {
+    const known = new UUIDSet(stored.map(r => r.ID));
+    const added = newlyPaused.filter(r => !known.Has(r.ID));
+    if (added.length === 0) return null;          // nothing new moved — any stored record stands as it is
+    return [...stored, ...added];
+}
+
+/**
+ * Which schedules a resume should re-activate.
+ *
+ * A recorded job is restored only when it STILL EXISTS and is STILL `Paused`. Both guards matter:
+ * a job deleted while the connection was paused must not be resurrected, and a job someone has
+ * already moved back to Active must not be written again for no reason. Jobs absent from the
+ * record are never touched, however they look now.
+ */
+export function DecideSchedulesToResume(
+    stored: ReadonlyArray<PausedScheduleRecord>,
+    jobs: ReadonlyArray<ScheduleJobState>
+): string[] {
+    const live = new Map(jobs.map(j => [j.ID, j]));
+    return stored
+        .filter(r => live.get(r.ID)?.Status === 'Paused')
+        .map(r => r.ID);
+}
+
+/** Reads the stored record out of a CompanyIntegration.Configuration JSON blob. */
+export function ReadPausedSchedules(configuration: string | null | undefined): PausedScheduleRecord[] {
+    if (!configuration) return [];
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(configuration);
+    } catch {
+        // A Configuration we cannot parse is not an error to raise here: pause/resume is not the
+        // operation that owns that blob. Report "no record" and let the rest of the flow proceed.
+        return [];
+    }
+    if (typeof parsed !== 'object' || parsed === null) return [];
+    const raw = (parsed as Record<string, unknown>)[PAUSED_SCHEDULES_KEY];
+    if (!Array.isArray(raw)) return [];
+    const out: PausedScheduleRecord[] = [];
+    for (const item of raw) {
+        if (typeof item !== 'object' || item === null) continue;
+        const rec = item as Record<string, unknown>;
+        const id = rec.ID;
+        const kind = rec.Kind;
+        if (typeof id !== 'string' || id.length === 0) continue;
+        out.push({ ID: id, Kind: kind === 'discovery' ? 'discovery' : 'sync' });
+    }
+    return out;
+}
+
+/**
+ * Returns the Configuration JSON with the record set (or cleared, when `record` is empty).
+ *
+ * Preserves every other key. An unparseable Configuration is REPLACED rather than merged — the
+ * alternative is refusing to pause because of unrelated corruption, and losing one broken blob
+ * is the better failure.
+ */
+export function WritePausedSchedules(
+    configuration: string | null | undefined,
+    record: ReadonlyArray<PausedScheduleRecord>
+): string {
+    let base: Record<string, unknown> = {};
+    if (configuration) {
+        try {
+            const parsed = JSON.parse(configuration);
+            if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
+                base = parsed as Record<string, unknown>;
+            }
+        } catch {
+            base = {};
+        }
+    }
+    if (record.length === 0) {
+        delete base[PAUSED_SCHEDULES_KEY];
+    } else {
+        base[PAUSED_SCHEDULES_KEY] = record.map(r => ({ ID: r.ID, Kind: r.Kind }));
+    }
+    return JSON.stringify(base);
+}
+
+/**
+ * How far a cancel request actually reaches — reported to the caller so the UI can say something
+ * TRUE rather than something reassuring.
+ *
+ * `durable`      the request is recorded where the owning process will observe it, whichever
+ *                process that is. "Stopping after the current batch" is honest.
+ * `this-process` a sync is running HERE and was signalled in-process. Honest, but only because
+ *                this happens to be the owner.
+ * `unknown`      a run is live and we could neither record the request durably nor find it in
+ *                this process. It will finish. What pause still guarantees is that no NEW sync
+ *                starts — say that, and do not claim the running one is stopping.
+ * `none`         nothing was running. There is nothing to say.
+ */
+export type CancelScope = 'durable' | 'this-process' | 'unknown' | 'none';
+
+/**
+ * Classifies the cancel from observable facts, not from which API was called.
+ *
+ * Deliberately derived rather than hardcoded per lineage: the durable path needs a run-row column
+ * that only exists on newer tenants, so the SAME code has to tell the truth on a tenant where
+ * that column is absent and the in-process registry is the only signal there is.
+ */
+export function DescribeCancelScope(facts: {
+    liveRunPresent: boolean;
+    durableRequestRecorded: boolean;
+    runningInThisProcess: boolean;
+}): CancelScope {
+    if (!facts.liveRunPresent) return 'none';
+    if (facts.durableRequestRecorded) return 'durable';
+    if (facts.runningInThisProcess) return 'this-process';
+    return 'unknown';
+}
+
+/** Operator-facing sentence for a cancel scope. Never claims a stop it cannot deliver. */
+export function DescribeCancelOutcome(scope: CancelScope): string {
+    switch (scope) {
+        case 'durable':
+        case 'this-process':
+            return 'A sync was running — it will stop after the current batch.';
+        case 'unknown':
+            return 'A sync is running in another process and will finish; no new sync will start.';
+        case 'none':
+            return '';
+    }
+}
+
+/** Operator-facing sentence for what pause did to the schedules. */
+export function DescribePauseOutcome(paused: ReadonlyArray<PausedScheduleRecord>, alreadyPaused: boolean): string {
+    if (alreadyPaused) return 'Schedules were already paused.';
+    if (paused.length === 0) return 'No active schedules to pause.';
+    const kinds = paused.map(p => p.Kind);
+    const parts: string[] = [];
+    if (kinds.includes('sync')) parts.push('data sync');
+    if (kinds.includes('discovery')) parts.push('schema refresh');
+    return `Paused the ${parts.join(' and ')} schedule${parts.length > 1 ? 's' : ''}.`;
+}

package/src/integration/CustomColumnPromoter.ts

@@ -47,7 +47,11 @@ import {
     type PromotionCandidate,
     type CustomKeyStat,
     type InferredColumnType,
+    BuildCatalogWriter,
+    WithCatalogScope,
 } from '@memberjunction/integration-engine';
+import { UUIDsEqual } from '@memberjunction/global';
+import { IntegrationEngineBase } from '@memberjunction/integration-engine-base';
 
 /**
  * A custom-key candidate persisted to CompanyIntegration.Configuration.customKeyCandidates
@@ -179,6 +183,27 @@ async function readAutoPromoteFlag(companyIntegrationID: string, user: UserInfo,
     } catch { return false; }
 }
 
+interface EntityPlan {
+    entityName: string;
+    entityInfo: EntityInfo;
+    entityMap: { ID: string; ExternalObjectName: string };
+    work: WorkItem[];
+    /** Index of this entity's input in the RSU batch; -1 when it needs no DDL. */
+    batchIndex: number;
+}
+
+/** What {@link IntegrationCustomColumnPromoter.PlanPromotion} worked out, before anything runs. */
+export interface PromotionPlan {
+    IntegrationID: string;
+    Plans: EntityPlan[];
+    /**
+     * The ADD COLUMN migrations, ready to run. The FIRST entry carries the durable post-restart
+     * PendingWork for the whole plan — RSU registers PendingWork from every input in a batch, so
+     * these can be appended to someone else's batch without losing it.
+     */
+    BatchInputs: RSUPipelineInput[];
+}
+
 const NOT_PROMOTED: SchemaPromotionResult = { Promoted: false, ColumnsAdded: [], SchemaUpdatePending: false };
 
 /**
@@ -199,30 +224,38 @@ export class IntegrationCustomColumnPromoter {
         return this.provider as unknown as DatabaseProviderBase;
     }
 
-    /** Entry point: promote custom columns for every entity touched by the sync. */
-    public async PromoteForSync(
+    /**
+     * PHASE 1 only: work out what WOULD be promoted, and the ADD COLUMN migrations that would do
+     * it, without running a pipeline.
+     *
+     * Split out so a caller that is ALREADY running an RSU batch can fold these migrations into
+     * it. A schema refresh that offers the accumulated overflow columns and then promotes them in
+     * its own second batch would restart the workspace twice in a row for one user action, and a
+     * restart storm is its own incident class here. One batch, one migration, one restart.
+     *
+     * Side effects are deliberate and belong to planning, not to promotion: stale already-promoted
+     * keys are purged from the staging JSON here, BEFORE any column is created and before the RSU
+     * pass that would restart this process. That runs even when nothing new is promoted, which is
+     * the only way residue on rows a sync never rewrites gets cleaned rather than endlessly
+     * re-detected as a phantom "new column".
+     *
+     * Returns null when there is nothing to do.
+     */
+    public async PlanPromotion(
         companyIntegrationID: string,
-        syncedEntityNames: string[],
+        entityNames: string[],
         customKeyStats?: Record<string, CustomKeyStat[]>,
-    ): Promise<SchemaPromotionResult> {
+    ): Promise<PromotionPlan | null> {
         const integrationID = await this.resolveIntegrationID(companyIntegrationID);
-        if (!integrationID) return NOT_PROMOTED;
+        if (!integrationID) return null;
 
         // ── PHASE 1: PLAN — build every entity's work list + its RSU input; run NO pipeline. ──
         // The old shape ran the FULL RSU pipeline per entity — a sync touching N entities with
         // candidates paid N CodeGen + compile passes where the batch API exists precisely to pay
         // one. Plan everything first, then promote once.
-        interface EntityPlan {
-            entityName: string;
-            entityInfo: EntityInfo;
-            entityMap: { ID: string; ExternalObjectName: string };
-            work: WorkItem[];
-            /** Index of this entity's input in the RSU batch; -1 when it needs no DDL. */
-            batchIndex: number;
-        }
         const plans: EntityPlan[] = [];
         const batchInputs: RSUPipelineInput[] = [];
-        for (const entityName of syncedEntityNames) {
+        for (const entityName of entityNames) {
             try {
                 const planned = await this.planWorkForEntity(companyIntegrationID, entityName, customKeyStats?.[entityName]);
                 if (!planned) continue; // no overflow column / no captured customs / no entity map
@@ -256,7 +289,7 @@ export class IntegrationCustomColumnPromoter {
                 LogError(`[CustomColumnPromoter] Planning failed for entity '${entityName}': ${this.msg(err)}`);
             }
         }
-        if (plans.length === 0) return NOT_PROMOTED;
+        if (plans.length === 0) return null;
 
         // Register the follow-up DURABLY before the restart, the way the apply-objects path does
         // (IntegrationDiscoveryResolver: `rsuInput.PendingWork = [pendingPayload]`). The restart is
@@ -290,6 +323,19 @@ export class IntegrationCustomColumnPromoter {
             }];
             batchInputs[0].ContextUser = this.user;
         }
+        return { IntegrationID: integrationID, Plans: plans, BatchInputs: batchInputs };
+    }
+
+    /** Entry point: promote custom columns for every entity touched by the sync. */
+    public async PromoteForSync(
+        companyIntegrationID: string,
+        syncedEntityNames: string[],
+        customKeyStats?: Record<string, CustomKeyStat[]>,
+    ): Promise<SchemaPromotionResult> {
+        const promotionPlan = await this.PlanPromotion(companyIntegrationID, syncedEntityNames, customKeyStats);
+        if (!promotionPlan) return NOT_PROMOTED;
+        const plans = promotionPlan.Plans;
+        const batchInputs = promotionPlan.BatchInputs;
 
         // ── PHASE 2: ONE batched RSU pass for ALL entities' ADD COLUMN migrations. ──
         // RunPipelineBatch runs every migration under one lock, then ONE CodeGen + compile +
@@ -300,7 +346,12 @@ export class IntegrationCustomColumnPromoter {
         // step with the database.
         let batchResults: RSUPipelineResult[] = [];
         if (batchInputs.length > 0) {
-            const batch = await RuntimeSchemaManager.Instance.RunPipelineBatch(batchInputs);
+            // Retrying variant. This runs UNATTENDED, mid-sync, with no operator watching: a
+            // transient ExecuteMigration / RunCodeGen / CompileTypeScript / RestartMJAPI failure
+            // drops the whole promotion, and the columns simply stay in overflow until something
+            // triggers a promotion again. The wrapper replays only those four steps and only while
+            // nothing in the batch was applied, so a partially-applied batch is still returned as-is.
+            const batch = await RuntimeSchemaManager.Instance.RunPipelineBatchWithRetry(batchInputs);
             batchResults = batch.Results ?? [];
             // M3: make the freshly-created EntityFields + regenerated sprocs visible in-process
             // ONCE for the whole batch. CRITICAL: the spread below builds sproc calls from this
@@ -350,7 +401,7 @@ export class IntegrationCustomColumnPromoter {
                     EntityName: plan.entityName,
                     EntityMapID: plan.entityMap.ID,
                     ExternalObjectName: plan.entityMap.ExternalObjectName,
-                    IntegrationID: integrationID,
+                    IntegrationID: promotionPlan.IntegrationID,
                     Columns: plan.work.map(w => ({
                         SourceKey: w.sourceKey,
                         ColumnName: w.columnName,
@@ -385,12 +436,19 @@ export class IntegrationCustomColumnPromoter {
      * Idempotent, because a pending row is marked Completed only after success and so stays
      * re-processable after a crash: field maps are filtered against those already active, and the
      * spread only fills a destination that is still empty.
+     *
+     * One entity's failure does not stop the others, but it is not swallowed either: after every
+     * entry has run, any failure is THROWN, so the caller fails the pending row instead of marking
+     * it Completed over columns that exist in the database and are never mapped.
+     *
+     * @throws PROMOTION_INCOMPLETE naming each entity that failed.
      */
     public async CompletePromotion(
         promoted: NonNullable<RSUPendingWork['PromotedColumns']>,
         knownEntities?: ReadonlyMap<string, EntityInfo>,
     ): Promise<Array<{ EntityName: string; ColumnName: string }>> {
         const columnsAdded: Array<{ EntityName: string; ColumnName: string }> = [];
+        const failures: string[] = [];
         for (const entry of promoted) {
             try {
                 // Provider first — post-restart it holds the REGENERATED class, which is the whole
@@ -399,6 +457,7 @@ export class IntegrationCustomColumnPromoter {
                 const entityInfo = this.provider.EntityByName(entry.EntityName) ?? knownEntities?.get(entry.EntityName);
                 if (!entityInfo) {
                     LogError(`[CustomColumnPromoter] Post-restart: entity '${entry.EntityName}' not found; skipping.`);
+                    failures.push(`${entry.EntityName}: entity not found`);
                     continue;
                 }
                 // Destination names are CARRIED, never recomputed: uniqueColumnName may have
@@ -424,17 +483,23 @@ export class IntegrationCustomColumnPromoter {
                     needsFieldMap: !existingMaps.has(c.SourceKey.toLowerCase()),
                 }));
 
-                await this.createIntegrationObjectFields(entry.IntegrationID, entry.ExternalObjectName, named);
+                await this.createIntegrationObjectFields(entry.IntegrationID, entry.EntityMapID, entry.ExternalObjectName, named);
                 await this.createFieldMaps(entry.EntityMapID, named.filter(n => n.needsFieldMap));
                 await this.spreadAndRebaseline(entry.EntityName, entry.EntityMapID, entityInfo, named);
 
                 LogStatus(`[CustomColumnPromoter] Post-restart: completed ${named.length} promoted column(s) on ${entry.EntityName}: ${named.map(n => n.columnName).join(', ')}`);
                 columnsAdded.push(...named.map(n => ({ EntityName: entry.EntityName, ColumnName: n.columnName })));
             } catch (err) {
-                // One entity must not abort the others; the pending row stays open for retry.
+                // One entity must not abort the others; the failure is thrown once they have all run.
                 LogError(`[CustomColumnPromoter] Post-restart completion failed for '${entry.EntityName}': ${this.msg(err)}`);
+                failures.push(`${entry.EntityName}: ${this.msg(err)}`);
             }
         }
+        if (failures.length > 0) {
+            throw new Error(
+                `PROMOTION_INCOMPLETE: ${failures.length} of ${promoted.length} entity(ies) failed to complete — ${failures.join('; ')}`
+            );
+        }
         return columnsAdded;
     }
 
@@ -781,13 +846,45 @@ export class IntegrationCustomColumnPromoter {
 
     /** Creates an IOF row per promoted field (IsCustom, MetadataSource='Discovered'). */
     private async createIntegrationObjectFields(
+        integrationID: string,
+        entityMapID: string,
+        externalObjectName: string,
+        named: WorkItem[],
+    ): Promise<void> {
+        // Resolve the CONNECTION from the entity map rather than hopping connection → integration
+        // and writing the shared rows. That hop is what let one tenant's promoted column appear in
+        // the catalog every other tenant on the same connector reads.
+        const entityMap = IntegrationEngineBase.Instance.EntityMaps.find(m => UUIDsEqual(m.ID, entityMapID));
+        const companyIntegration = entityMap
+            ? IntegrationEngineBase.Instance.GetCompanyIntegrationByID(entityMap.CompanyIntegrationID)
+            : undefined;
+        if (!companyIntegration) {
+            LogError(`[CustomColumnPromoter] Could not resolve the connection behind entity map ${entityMapID}; catalog rows skipped for '${externalObjectName}'.`);
+            return;
+        }
+        // In the connection's own scope, entered here per entry: the post-restart consumer calls
+        // in with no scope, and unscoped a connection with per-connection rows reads as having none,
+        // so BuildCatalogWriter throws PER_CONNECTION_CATALOG_MISSING. Per entry, so a batch that
+        // spans connections resolves each one to its own catalog.
+        await WithCatalogScope(
+            companyIntegration.ID,
+            () => this.writeIntegrationObjectFields(companyIntegration, integrationID, externalObjectName, named),
+            this.user,
+        );
+    }
+
+    /** The catalog writes of {@link createIntegrationObjectFields}, run inside the connection's scope. */
+    private async writeIntegrationObjectFields(
+        companyIntegration: MJCompanyIntegrationEntity,
         integrationID: string,
         externalObjectName: string,
         named: WorkItem[],
     ): Promise<void> {
-        const objectID = await this.resolveIntegrationObjectID(integrationID, externalObjectName);
+        const writer = BuildCatalogWriter(this.provider, companyIntegration, this.user);
+        const objectID = (await writer.ObjectsInScope())
+            .find(o => o.Name.toLowerCase() === externalObjectName.toLowerCase())?.ID ?? null;
         if (!objectID) {
-            LogError(`[CustomColumnPromoter] No IntegrationObject '${externalObjectName}' for integration ${integrationID}; IOF rows skipped.`);
+            LogError(`[CustomColumnPromoter] No catalog object '${externalObjectName}' for integration ${integrationID}; catalog field rows skipped.`);
             return;
         }
         // Lookup-or-reactivate-or-create — NEVER blind-create over an existing field. A field the source
@@ -795,13 +892,16 @@ export class IntegrationCustomColumnPromoter {
         // key reappears in the payload and reaches promotion, the IOF already exists: reactivate it (so the
         // active-filtered ApplyAll re-materializes the still-present column) instead of creating a duplicate.
         // This is the "removed-then-re-added" case + the recovery case (column existed, field map missing).
-        const existingIOFs = await this.existingIOFsByName(objectID);
+        const existingIOFs = new Map<string, { ID: string; Status: string }>();
+        for (const f of await writer.FieldsForObject(objectID)) {
+            existingIOFs.set((f.Name ?? '').toLowerCase(), { ID: f.ID, Status: f.Status ?? 'Active' });
+        }
         for (const n of named) {
             const existing = existingIOFs.get(n.sourceKey.toLowerCase());
             if (existing) {
                 if (existing.Status !== 'Active') {
-                    const reIof = await this.provider.GetEntityObject<MJIntegrationObjectFieldEntity>('MJ: Integration Object Fields', this.user);
-                    if (await reIof.Load(existing.ID)) {
+                    const reIof = await writer.LoadField(existing.ID);
+                    if (reIof) {
                         reIof.Status = 'Active';
                         const ok = await reIof.Save();
                         if (!ok) LogError(`[CustomColumnPromoter] Failed to reactivate IOF '${n.sourceKey}': ${reIof.LatestResult?.CompleteMessage ?? 'unknown'}`);
@@ -809,8 +909,10 @@ export class IntegrationCustomColumnPromoter {
                 }
                 continue;
             }
-            const iof = await this.provider.GetEntityObject<MJIntegrationObjectFieldEntity>('MJ: Integration Object Fields', this.user);
-            iof.NewRecord();
+            const iof = await writer.NewFieldRow();
+            // An overflow-promoted column was seen in live data, never described by an endpoint and
+            // never declared — so its provenance is 'Sampled', and it has no declared row behind it.
+            writer.StampNewField(iof, null, 'Sampled');
             iof.IntegrationObjectID = objectID;
             iof.Name = n.sourceKey;
             iof.DisplayName = n.sourceKey;
@@ -1102,40 +1204,13 @@ export class IntegrationCustomColumnPromoter {
             ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND Entity='${this.escape(entityName)}'`,
             ResultType: 'simple',
             MaxRows: 1,
+            SkipTotalRowCount: true,
         }, this.user);
         const row = res.Success ? res.Results?.[0] : undefined;
         return row ? { ID: row.ID, ExternalObjectName: row.ExternalObjectName } : null;
     }
 
-    /** Existing IOF field names (lowercased) for an object — for idempotent IOF creation. */
-    private async existingIOFsByName(objectID: string): Promise<ReadonlyMap<string, { ID: string; Status: string }>> {
-        const rv = new RunView();
-        const res = await rv.RunView<MJIntegrationObjectFieldEntity>({
-            EntityName: 'MJ: Integration Object Fields',
-            ExtraFilter: `IntegrationObjectID='${objectID}'`,
-            Fields: ['ID', 'Name', 'Status'],
-            ResultType: 'simple',
-        }, this.user);
-        const map = new Map<string, { ID: string; Status: string }>();
-        if (res.Success) for (const r of res.Results ?? []) {
-            map.set((r.Name ?? '').toLowerCase(), { ID: String(r.ID), Status: r.Status ?? 'Active' });
-        }
-        return map;
-    }
 
-    /** Resolves the IntegrationObject ID for an external object name under an integration. */
-    private async resolveIntegrationObjectID(integrationID: string, externalObjectName: string): Promise<string | null> {
-        const rv = new RunView();
-        const res = await rv.RunView<MJIntegrationObjectEntity>({
-            EntityName: 'MJ: Integration Objects',
-            ExtraFilter: `IntegrationID='${integrationID}' AND Name='${this.escape(externalObjectName)}'`,
-            Fields: ['ID', 'Name', 'IntegrationID'],
-            ResultType: 'simple',
-            MaxRows: 1,
-        }, this.user);
-        const row = res.Success ? res.Results?.[0] : undefined;
-        return row ? row.ID : null;
-    }
 
     private escape(value: string): string {
         return value.replace(/'/g, "''");