@memberjunction/integration-engine 5.48.0 → 5.50.0

package/dist/IntegrationEngine.js

@@ -1,4 +1,4 @@
-import { CompositeKey, LogStatusEx, Metadata, RunView } from '@memberjunction/core';
+import { CompositeKey, DatabaseProviderBase, LogStatusEx, Metadata, RunView } from '@memberjunction/core';
 import { BaseSingleton, UUIDsEqual } from '@memberjunction/global';
 import { IntegrationEngineBase } from '@memberjunction/integration-engine-base';
 import { ClassifyError, IsRetryableError } from './types.js';
@@ -9,10 +9,11 @@ import { FieldMappingEngine } from './FieldMappingEngine.js';
 import { MatchEngine } from './MatchEngine.js';
 import { WatermarkService } from './WatermarkService.js';
 import { SyncLogger } from './SyncLogger.js';
-import { CONTENT_HASH_COLUMN, computeContentHashWithOverflow, contentHashBasis } from './ContentHash.js';
-import { buildContentHashPrefetchFilter } from './prefetchFilter.js';
+import { CONTENT_HASH_COLUMN, computeContentHash } from './ContentHash.js';
+import { RecordMapBatch } from './RecordMapBatch.js';
+import { buildContentHashPrefetchFilter, quoteTextLiteral } from './prefetchFilter.js';
 import { serializeKeyValue } from './KeySerialization.js';
-import { CUSTOM_OVERFLOW_COLUMN, reconcileOverflowValue } from './CustomOverflow.js';
+import { CUSTOM_OVERFLOW_COLUMN, reconcileOverflowValue, foldCustomKeyStats } from './CustomOverflow.js';
 import { partitionRecords, partitionRollupHash, diffPartitions, partitionKeyForIdentity } from './HashDiff.js';
 import { RateLimiter } from './RateLimiter.js';
 import { AdaptiveConcurrencyController, RunAdaptive } from './AdaptiveConcurrency.js';
@@ -191,6 +192,30 @@ export class IntegrationEngine extends BaseSingleton {
         }
         return ceiling;
     }
+    /**
+     * Hard ceiling on the record-map keyset page size. Each page is one round trip that materializes
+     * this many rows in memory; past 50k the per-page cost stops amortizing and starts risking the
+     * response-size limits of the transport.
+     */
+    static { this.RECORD_MAP_PAGE_SIZE_CEILING = 50_000; }
+    /**
+     * Page size for the keyset walk over MJ: Company Integration Record Maps (see LoadAllRecordMaps).
+     * Read from MJ_INTEGRATION_RECORD_MAP_PAGE_SIZE at class-init and clamped to
+     * [1, RECORD_MAP_PAGE_SIZE_CEILING]; defaults to 10,000.
+     */
+    static { this.RecordMapPageSize = IntegrationEngine.computeRecordMapPageSize(); }
+    /** Reads + clamps the record-map page size from env; logs once when a configured value is clamped. */
+    static computeRecordMapPageSize() {
+        const ceiling = IntegrationEngine.RECORD_MAP_PAGE_SIZE_CEILING;
+        const raw = parseInt(process.env.MJ_INTEGRATION_RECORD_MAP_PAGE_SIZE ?? '', 10);
+        if (!Number.isFinite(raw) || raw <= 0)
+            return 10_000;
+        if (raw > ceiling) {
+            LogStatusEx({ message: `[IntegrationEngine] MJ_INTEGRATION_RECORD_MAP_PAGE_SIZE=${raw} exceeds the maximum ${ceiling} — clamped to ${ceiling}.` });
+            return ceiling;
+        }
+        return raw;
+    }
     /** Registers (or clears, with undefined) the post-sync custom-column promotion hook. */
     SetPostSyncSchemaPromotionCallback(callback) {
         this.postSyncSchemaPromotionCallback = callback;
@@ -201,6 +226,36 @@ export class IntegrationEngine extends BaseSingleton {
     }
     /** In-process lock map to prevent concurrent syncs for the same CompanyIntegration */
     static { this.activeSyncs = new Map(); }
+    /**
+     * Maintenance locks: while a metadata refresh / schema evolution / RSU pipeline is
+     * running for a CompanyIntegration, data syncs MUST NOT start ("locks of sync and scheduled
+     * sync must occur" — the refresh is rewriting the very metadata, field maps and DDL the sync
+     * would read). Held per-CI; RunSync refuses with a clear error while held, and the scheduled
+     * drivers skip with a logged reason. Keyed lowercase like activeSyncs.
+     */
+    static { this.maintenanceLocks = new Map(); }
+    /**
+     * Acquires the maintenance lock for a CompanyIntegration. Returns false (does NOT wait) when
+     * a data sync is currently running or another maintenance operation already holds the lock —
+     * the caller decides whether to wait for `GetSyncProgress` to clear or surface the conflict.
+     */
+    static AcquireMaintenanceLock(companyIntegrationID, reason) {
+        const key = companyIntegrationID.toLowerCase();
+        if (IntegrationEngine.activeSyncs.has(key))
+            return false; // a sync is mid-flight
+        if (IntegrationEngine.maintenanceLocks.has(key))
+            return false; // refresh already running
+        IntegrationEngine.maintenanceLocks.set(key, { Reason: reason, AcquiredAt: new Date() });
+        return true;
+    }
+    /** Releases the maintenance lock (idempotent — safe in a finally). */
+    static ReleaseMaintenanceLock(companyIntegrationID) {
+        IntegrationEngine.maintenanceLocks.delete(companyIntegrationID.toLowerCase());
+    }
+    /** Current maintenance lock for a CompanyIntegration, or undefined when none is held. */
+    static GetMaintenanceLock(companyIntegrationID) {
+        return IntegrationEngine.maintenanceLocks.get(companyIntegrationID.toLowerCase());
+    }
     runWriteExclusive(fn) {
         // Run fn after the prior write completes (whether it resolved or rejected); keep the chain
         // alive past failures so one errored batch never deadlocks subsequent writers.
@@ -231,6 +286,17 @@ export class IntegrationEngine extends BaseSingleton {
     static GetAllSyncProgress() {
         return new Map(IntegrationEngine._syncProgress);
     }
+    /**
+     * U3 — pure, MONOTONIC progress fold: applies one per-map progress event to the live
+     * snapshot as a high-water mark. Under syncConcurrency > 1 events arrive out of order,
+     * so counters only ever ratchet UP — a progress bar must never go backwards. Totals are
+     * assigned (they're authoritative per event); completed/processed take max().
+     */
+    static RatchetProgressSnapshot(entry, progress) {
+        entry.EntityMapsTotal = progress.TotalEntityMaps;
+        entry.EntityMapsCompleted = Math.max(entry.EntityMapsCompleted, progress.EntityMapIndex);
+        entry.RecordsProcessed = Math.max(entry.RecordsProcessed, progress.RecordsProcessedInCurrentMap);
+    }
     /**
      * Resumes any syncs that were orphaned by a process restart.
      * Finds all CompanyIntegrationRun records with Status='In Progress',
@@ -366,6 +432,20 @@ export class IntegrationEngine extends BaseSingleton {
             console.warn(`[IntegrationEngine] Sync already running for ${lockKey}, waiting...`);
             return existing;
         }
+        // A metadata refresh / schema evolution / RSU pipeline holds the maintenance
+        // lock — a sync starting mid-refresh would read half-rewritten metadata/field maps/DDL.
+        // Refuse loudly (no queueing: the refresh may restart the process; the caller/schedule
+        // simply retries after it completes).
+        const maintenance = IntegrationEngine.maintenanceLocks.get(lockKey);
+        if (maintenance) {
+            const message = `Sync refused: ${maintenance.Reason} is in progress for this connection (since ${maintenance.AcquiredAt.toISOString()}). Retry after it completes.`;
+            console.warn(`[IntegrationEngine] ${message}`);
+            return {
+                Success: false, ErrorMessage: message, RecordsProcessed: 0, RecordsCreated: 0,
+                RecordsUpdated: 0, RecordsDeleted: 0, RecordsErrored: 0, RecordsSkipped: 0,
+                Errors: [], EntityMapResults: [], Duration: 0,
+            };
+        }
         // Initialize abort controller and progress tracking
         const abortController = new AbortController();
         IntegrationEngine._abortControllers.set(lockKey, abortController);
@@ -380,14 +460,14 @@ export class IntegrationEngine extends BaseSingleton {
             RecordsErrored: 0,
             TriggerType: triggerType,
         });
-        // Wrap caller's onProgress with internal tracking
+        // Wrap caller's onProgress with internal tracking. U3 — MONOTONIC: with
+        // syncConcurrency > 1 the per-map events arrive out of order (map 3 can emit after
+        // map 7), so raw assignment made the progress bar go BACKWARDS. The snapshot is a
+        // high-water mark, so only ever ratchet the counters upward.
         const wrappedProgress = (progress) => {
             const entry = IntegrationEngine._syncProgress.get(lockKey);
-            if (entry) {
-                entry.EntityMapsTotal = progress.TotalEntityMaps;
-                entry.EntityMapsCompleted = progress.EntityMapIndex;
-                entry.RecordsProcessed = progress.RecordsProcessedInCurrentMap;
-            }
+            if (entry)
+                IntegrationEngine.RatchetProgressSnapshot(entry, progress);
             if (onProgress)
                 onProgress(progress);
         };
@@ -972,7 +1052,18 @@ export class IntegrationEngine extends BaseSingleton {
             if (cfgRaw) {
                 try {
                     const cfg = JSON.parse(cfgRaw);
-                    for (const name of [cfg.parentObjectName, cfg.ReferencedType]) {
+                    const softParentNames = [cfg.parentObjectName, cfg.ReferencedType];
+                    // MULTI-LEVEL template-var children declare a per-var parent MAP
+                    // (Configuration.parentObjectNames = {"<var>":"<SiblingObject>"}) instead of the
+                    // single parentObjectName. Include those parents too, or a `/a/{x}/b/{y}/c` child
+                    // orders after only ONE of its parents — the sync DAG must gate it behind ALL of
+                    // them (parents-populated-before-child, the DAG ordering). Matches what the wizard's
+                    // DependsOn exposes, so UI hint and sync ordering agree. Additive edges only
+                    // (cycle-guarded downstream), so this can only make ordering MORE correct.
+                    if (cfg.parentObjectNames && typeof cfg.parentObjectNames === 'object' && !Array.isArray(cfg.parentObjectNames)) {
+                        softParentNames.push(...Object.values(cfg.parentObjectNames).filter((v) => typeof v === 'string'));
+                    }
+                    for (const name of softParentNames) {
                         const parent = name ? ioByName.get(name.toLowerCase()) : undefined;
                         if (parent && parent !== ioId && selectedIoIds.has(parent))
                             set.add(parent);
@@ -1390,6 +1481,11 @@ export class IntegrationEngine extends BaseSingleton {
         const fetchedExternalIDs = new Set(); // Track all IDs seen during this pull for orphan detection
         let orphanTrackingOverflowed = false; // set if the ID set exceeds ORPHAN_DETECTION_MAX_IDS → skip the sweep, don't OOM
         const accumulatedMapped = []; // partition-reconcile mode: collect mapped records, apply post-loop
+        // Custom-key stats: in-memory aggregation of every UNMAPPED source key seen this
+        // run, independent of whether the row is written or content-hash-skipped. Bounded memory:
+        // one entry per distinct key + a capped value sample. See CustomKeyStat.
+        const customKeyAgg = new Map();
+        let customKeyTotalRecords = 0;
         while (hasMore) {
             if (abortSignal?.aborted) {
                 console.log(`[IntegrationEngine] Sync cancelled for ${entityMap.ExternalObjectName} after ${recordsInMap} records — saving watermark`);
@@ -1538,6 +1634,11 @@ export class IntegrationEngine extends BaseSingleton {
                 }
             }
             const mapped = this.fieldMappingEngine.Apply(batch.Records, fieldMaps, entityMap.Entity);
+            // Custom-key stats: aggregate unmapped keys for EVERY mapped record here —
+            // before any skip decision — so candidates + sizing stats exist even when the
+            // content-hash fast path skips the row (the hash basis deliberately excludes them).
+            foldCustomKeyStats(mapped.map(r => r.UnmappedFields), customKeyAgg);
+            customKeyTotalRecords += mapped.length;
             // Partition (Merkle) reconcile defers match + apply: accumulate mapped records now; the
             // partition-diff + selective apply runs once after the full fetch (applyViaPartitionReconcile).
             if (partitionReconcile) {
@@ -1740,6 +1841,19 @@ export class IntegrationEngine extends BaseSingleton {
                 `the result set is INCOMPLETE. The watermark was held, so the skipped window is re-fetched next run; ` +
                 `records on the reachable pages were synced normally.`, { skipped: true });
         }
+        // Surface the run's custom-key statistics (out-of-band candidates). Keyed by the
+        // target MJ entity so the post-sync promotion callback can line them up with its scan.
+        if (customKeyAgg.size > 0) {
+            result.CustomKeyStats = {
+                [entityMap.Entity]: [...customKeyAgg.entries()].map(([key, s]) => ({
+                    Key: key,
+                    Occurrences: s.occurrences,
+                    TotalRecords: customKeyTotalRecords,
+                    MaxLength: s.maxLength,
+                    SampleValues: s.samples,
+                })).sort((a, b) => a.Key.localeCompare(b.Key)),
+            };
+        }
         await this.CreateRunDetail(run, entityMap, result, contextUser);
         return result;
     }
@@ -1931,19 +2045,21 @@ export class IntegrationEngine extends BaseSingleton {
                 `use incremental push (set a push watermark) or raise MJ_INTEGRATION_FULL_PUSH_MAX_RECORDS. ` +
                 `Refusing to partial-push (that would advance the watermark past unsent rows and drop data).`);
         }
-        // Load existing record maps to know which records already exist externally
-        const mapResult = await rv.RunView({
-            EntityName: 'MJ: Company Integration Record Maps',
-            ExtraFilter: `CompanyIntegrationID='${companyIntegration.ID}' AND EntityID='${entityMap.EntityID}'`,
-            Fields: ['EntityRecordID', 'ExternalSystemRecordID'],
-            ResultType: 'simple',
-            BypassCache: true, // sync decisions must reflect committed record-map state, not a stale cache
-        }, contextUser);
+        // Load existing record maps to know which records already exist externally.
+        // Paged: an unpaged read is silently capped at the entity's UserViewMaxRows (1000 by
+        // default), and a truncated map here makes already-synced records look brand new — so
+        // the push would re-CREATE them externally as duplicates.
+        const allMaps = await this.LoadAllRecordMaps(companyIntegration.ID, entityMap.EntityID, contextUser);
+        if (!allMaps.Complete) {
+            // Refuse rather than push a partial picture: with an incomplete map, every unmapped
+            // record reads as "not yet in the external system" and gets created a second time.
+            throw new Error(`Cannot push ${entityMap.Entity}: failed to load the existing record map ` +
+                `(${allMaps.Error ?? 'unknown error'}). Refusing to push — an incomplete map would ` +
+                `re-create already-synced records as duplicates in the external system.`);
+        }
         const existingMaps = new Map();
-        if (mapResult.Success) {
-            for (const m of mapResult.Results) {
-                existingMaps.set(m.EntityRecordID, m.ExternalSystemRecordID);
-            }
+        for (const m of allMaps.Rows) {
+            existingMaps.set(m.EntityRecordID, m.ExternalSystemRecordID);
         }
         const md = this.ProviderToUse;
         const entityInfo = md.EntityByName(entityMap.Entity);
@@ -2229,18 +2345,20 @@ export class IntegrationEngine extends BaseSingleton {
      * These records were deleted externally and should be removed from MJ.
      */
     async DeleteOrphanedRecords(companyIntegration, entityMap, fetchedExternalIDs, result, contextUser, logger) {
-        const rv = new RunView();
-        const mapResult = await rv.RunView({
-            EntityName: 'MJ: Company Integration Record Maps',
-            ExtraFilter: `CompanyIntegrationID='${companyIntegration.ID}' ` +
-                `AND EntityID='${entityMap.EntityID}'`,
-            Fields: ['EntityRecordID', 'ExternalSystemRecordID'],
-            ResultType: 'simple',
-            BypassCache: true, // orphan-sweep compares against committed record-map state
-        }, contextUser);
-        if (!mapResult.Success)
+        // Paged: an unpaged read is silently capped at the entity's UserViewMaxRows (1000 by
+        // default). A tenant with 5,000 orphans would clear 1,000 per run and the operator would
+        // see a clean run every time — the truncation was invisible, which is the actual bug.
+        const allMaps = await this.LoadAllRecordMaps(companyIntegration.ID, entityMap.EntityID, contextUser);
+        if (!allMaps.Complete) {
+            // Deleting against a partial map is the dangerous direction: rows we simply failed to
+            // read are indistinguishable from rows the external system dropped. Skip the sweep and
+            // say so, rather than archiving live records on incomplete evidence.
+            logger?.warning(entityMap.ExternalObjectName ?? entityMap.ID, 'ORPHAN_SWEEP_SKIPPED', `Delete-detection was skipped for ${entityMap.Entity}: the record map could not be read completely ` +
+                `(${allMaps.Error ?? 'unknown error'}) after ${allMaps.Rows.length} row(s). No records were deleted. ` +
+                `Deleting on a partial map would archive live records.`, { rowsRead: allMaps.Rows.length });
             return;
-        const orphans = mapResult.Results.filter(m => !fetchedExternalIDs.has(m.ExternalSystemRecordID));
+        }
+        const orphans = allMaps.Rows.filter(m => !fetchedExternalIDs.has(m.ExternalSystemRecordID));
         if (orphans.length === 0)
             return;
         console.log(`[IntegrationEngine] Orphan detection for ${entityMap.ExternalObjectName}: ${orphans.length} records in MJ not found in external system`);
@@ -2368,7 +2486,9 @@ export class IntegrationEngine extends BaseSingleton {
         const buckets = partitionRecords(mappedRecords, idOf, partitionOf);
         const newRollups = new Map();
         for (const [partition, recs] of buckets) {
-            newRollups.set(partition, partitionRollupHash(recs, r => contentHashBasis(r.MappedFields, r.UnmappedFields)));
+            // Content-hash basis: MAPPED fields only — an unmapped/custom key must never move a
+            // partition rollup (its capture + promotion is handled out-of-band via CustomKeyStats).
+            newRollups.set(partition, partitionRollupHash(recs, r => r.MappedFields));
         }
         // Diff against last sync's snapshot; only changed/added partitions need a deep apply. On a FORCED
         // FULL SYNC, treat the snapshot as empty so EVERY partition is re-applied: fullSync is the operator's
@@ -2449,6 +2569,11 @@ export class IntegrationEngine extends BaseSingleton {
         // to 500 healthy records.
         const APPLY_BATCH_SIZE = 500;
         const provider = this.ProviderToUse;
+        // One batched record-map writer for this entity map's whole apply pass. Every path that
+        // used to spend three round trips per record establishing a mapping now queues into this
+        // and the queue is flushed set-based once the batch has settled — the single largest cost
+        // in a no-change sync.
+        const recordMaps = new RecordMapBatch(this.ProviderToUse, companyIntegration.ID, contextUser, (ciID, extID, entID, recID, user) => this.SaveRecordMap(ciID, extID, entID, recID, user));
         for (let i = 0; i < records.length; i += APPLY_BATCH_SIZE) {
             const batch = records.slice(i, i + APPLY_BATCH_SIZE);
             const batchStartProcessed = result.RecordsProcessed;
@@ -2475,7 +2600,7 @@ export class IntegrationEngine extends BaseSingleton {
                     try {
                         for (const record of batch) {
                             result.RecordsProcessed++;
-                            await this.ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds);
+                            await this.ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps);
                         }
                         await provider.CommitTransaction();
                     }
@@ -2484,6 +2609,9 @@ export class IntegrationEngine extends BaseSingleton {
                         // The batch transaction rolled back; the skip-IDs collected during the failed attempt
                         // never committed. Reset and let the per-record retry re-collect only what commits.
                         reconciledSkipIds = [];
+                        // Same reasoning for the queued mappings — the records they point at were rolled
+                        // back, so writing them would leave the map referencing rows that don't exist.
+                        recordMaps.Discard();
                         // Roll back the in-memory counters that ApplySingleRecord bumped inside the failed batch
                         result.RecordsProcessed = batchStartProcessed;
                         result.RecordsCreated = batchStartCreated;
@@ -2499,7 +2627,7 @@ export class IntegrationEngine extends BaseSingleton {
                         }
                         // Degrade to per-record application so the failure isolates to the poison
                         // record(s) and every good record in this batch still commits.
-                        await this.applyRecordsIndividually(batch, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds);
+                        await this.applyRecordsIndividually(batch, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps);
                     }
                 }
                 else {
@@ -2514,7 +2642,7 @@ export class IntegrationEngine extends BaseSingleton {
                         try {
                             // §10 — bounded inline retry for provably-transient save failures (auto-commit per
                             // record, so no transaction to manage); permanent errors throw straight to dead-letter.
-                            await WithRetry(() => this.ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds), undefined, (e) => !(e instanceof SchemaNotGeneratedError) && IsRetryableError(ClassifyError(e).Code), (attempt, e, delayMs) => logger?.emit('sync.record.retry', {
+                            await WithRetry(() => this.ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps), undefined, (e) => !(e instanceof SchemaNotGeneratedError) && IsRetryableError(ClassifyError(e).Code), (attempt, e, delayMs) => logger?.emit('sync.record.retry', {
                                 phase: 'save',
                                 externalObjectName: entityMap.ExternalObjectName,
                                 externalId: record.ExternalRecord?.ExternalID ?? '',
@@ -2544,9 +2672,35 @@ export class IntegrationEngine extends BaseSingleton {
                 if (reconciledSkipIds.length > 0) {
                     await this.TouchLastReconciledAt(entityMap, reconciledSkipIds, contextUser, logger);
                 }
+                // Write the batch's record maps set-based, now that the records they point at are
+                // committed. Deliberately AFTER the transaction rather than inside it: the mapping
+                // is derived data that the next sync can re-establish by primary key, and keeping
+                // it out of the write transaction keeps that transaction as short as possible.
+                await this.FlushRecordMaps(recordMaps, entityMap, logger);
             });
         }
     }
+    /**
+     * Flushes the queued record maps and reports any that failed, per external ID.
+     *
+     * Failures here are warnings, not record errors: the record itself saved. A missing mapping
+     * degrades orphan detection until the next sync re-establishes it by primary key, which is
+     * worth telling the operator about but is not a reason to mark the record — or the run —
+     * failed. Flushing itself never throws for the same reason.
+     */
+    async FlushRecordMaps(recordMaps, entityMap, logger) {
+        try {
+            await recordMaps.Flush();
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            console.warn(`[IntegrationEngine] Record-map flush failed for ${entityMap.ExternalObjectName}: ${msg}`);
+            logger?.warning(entityMap.ExternalObjectName ?? entityMap.Entity ?? entityMap.ID, 'RECORD_MAP_FLUSH_FAILED', msg, {});
+        }
+        for (const failure of recordMaps.TakeFailures()) {
+            logger?.warning(entityMap.ExternalObjectName ?? entityMap.Entity ?? entityMap.ID, 'RECORD_MAP_WRITE_FAILED', `Record map not written for external ID ${failure.ExternalID}: ${failure.ErrorMessage}`, { externalId: failure.ExternalID, entityId: failure.EntityID });
+        }
+    }
     /**
      * Refreshes __mj_integration_LastReconciledAt = now for a set of records that the content-hash
      * fast path skipped (present + confirmed-unchanged on the source). Issues ONE set-based UPDATE
@@ -2606,7 +2760,7 @@ export class IntegrationEngine extends BaseSingleton {
      *
      * Begin/Commit/Rollback are always matched per record (no leaked open transaction).
      */
-    async applyRecordsIndividually(batch, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds) {
+    async applyRecordsIndividually(batch, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps) {
         const provider = this.ProviderToUse;
         for (const record of batch) {
             result.RecordsProcessed++;
@@ -2619,7 +2773,7 @@ export class IntegrationEngine extends BaseSingleton {
                 await WithRetry(async () => {
                     await provider.BeginTransaction();
                     try {
-                        await this.ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds);
+                        await this.ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps);
                         await provider.CommitTransaction();
                     }
                     catch (e) {
@@ -2674,7 +2828,7 @@ export class IntegrationEngine extends BaseSingleton {
     /**
      * Applies a single record change (Create, Update, Delete, or Skip).
      */
-    async ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds) {
+    async ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps) {
         logger?.emit('sync.record.decision', {
             externalId: record.ExternalRecord.ExternalID,
             objectType: record.ExternalRecord.ObjectType,
@@ -2690,7 +2844,7 @@ export class IntegrationEngine extends BaseSingleton {
         try {
             switch (record.ChangeType) {
                 case 'Create': {
-                    const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser);
+                    const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps);
                     if (outcome === 'updated')
                         result.RecordsUpdated++;
                     else if (outcome === 'skipped')
@@ -2700,7 +2854,7 @@ export class IntegrationEngine extends BaseSingleton {
                     break;
                 }
                 case 'Update':
-                    await this.UpdateRecord(record, companyIntegration, entityMap, result, contextUser, precheckHashes, reconciledSkipIds);
+                    await this.UpdateRecord(record, companyIntegration, entityMap, result, contextUser, precheckHashes, reconciledSkipIds, recordMaps);
                     break;
                 case 'Delete': {
                     const didDelete = await this.DeleteRecord(record, entityMap, contextUser);
@@ -2760,7 +2914,7 @@ export class IntegrationEngine extends BaseSingleton {
      *
      * @returns true if an existing row was updated, false if a new row was inserted (so the caller counts correctly).
      */
-    async CreateRecord(record, companyIntegration, entityMap, contextUser) {
+    async CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps) {
         const md = this.ProviderToUse;
         const entity = await md.GetEntityObject(record.MJEntityName, contextUser);
         const entityInfo = md.EntityByName(record.MJEntityName);
@@ -2786,8 +2940,8 @@ export class IntegrationEngine extends BaseSingleton {
             if (hasHashColumn) {
                 const storedHash = entity.Get(CONTENT_HASH_COLUMN);
                 if (typeof storedHash === 'string' && storedHash.length > 0
-                    && storedHash === computeContentHashWithOverflow(record.MappedFields ?? {}, record.UnmappedFields)) {
-                    await this.SaveRecordMap(companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|'), contextUser);
+                    && storedHash === computeContentHash(record.MappedFields ?? {})) {
+                    await this.QueueRecordMap(recordMaps, companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|'), contextUser);
                     return 'skipped';
                 }
             }
@@ -2796,8 +2950,8 @@ export class IntegrationEngine extends BaseSingleton {
             // re-establish the possibly-cleared record map and SKIP the write — leaving __mj_UpdatedAt
             // and the integration LastSynced columns untouched, exactly like the content-hash skip path.
             this.SetEntityFields(entity, record.MappedFields);
-            if (!entity.Dirty) {
-                await this.SaveRecordMap(companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|'), contextUser);
+            if (!entity.Dirty && !this.needsSyncStateRepair(entity, entityInfo)) {
+                await this.QueueRecordMap(recordMaps, companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|'), contextUser);
                 return 'skipped';
             }
         }
@@ -2822,7 +2976,7 @@ export class IntegrationEngine extends BaseSingleton {
         // incremental sync. SaveRecordMap is an upsert keyed on (CompanyIntegration, Entity, ExternalID),
         // so this also re-establishes a map that was previously cleared.
         const entityRecordID = entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|');
-        await this.SaveRecordMap(companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entityRecordID, contextUser);
+        await this.QueueRecordMap(recordMaps, companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entityRecordID, contextUser);
         return existed ? 'updated' : 'created';
     }
     /**
@@ -2854,10 +3008,10 @@ export class IntegrationEngine extends BaseSingleton {
      * If the record cannot be loaded (e.g. it was deleted or never fully created),
      * falls back to CreateRecord (upsert behavior).
      */
-    async UpdateRecord(record, companyIntegration, entityMap, result, contextUser, precheckHashes, reconciledSkipIds) {
+    async UpdateRecord(record, companyIntegration, entityMap, result, contextUser, precheckHashes, reconciledSkipIds, recordMaps) {
         if (!record.MatchedMJRecordID) {
             // No matched ID — upsert by PK (insert; or update/skip if the PK already exists)
-            const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser);
+            const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps);
             if (outcome === 'updated')
                 result.RecordsUpdated++;
             else if (outcome === 'skipped')
@@ -2873,7 +3027,7 @@ export class IntegrationEngine extends BaseSingleton {
         // below is the fallback for entities without the hash column.
         if (precheckHashes) {
             const stored = precheckHashes.get(record.MatchedMJRecordID);
-            if (stored && stored === computeContentHashWithOverflow(record.MappedFields ?? {}, record.UnmappedFields)) {
+            if (stored && stored === computeContentHash(record.MappedFields ?? {})) {
                 result.RecordsSkipped++;
                 // Re-establish the external↔MJ record map even on the content-hash skip. A record can
                 // reach UpdateRecord matched by KEY FIELDS / PK (MatchEngine.FindByKeyFields queries the
@@ -2886,7 +3040,7 @@ export class IntegrationEngine extends BaseSingleton {
                 // (CompanyIntegration, Entity, ExternalID) — idempotent for already-mapped records, and the
                 // CreateRecord skip branches already do exactly this. MatchedMJRecordID IS the dest PK
                 // (PrimaryKeys order, '|'-joined), which is the EntityRecordID the map stores.
-                await this.SaveRecordMap(companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, record.MatchedMJRecordID, contextUser);
+                await this.QueueRecordMap(recordMaps, companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, record.MatchedMJRecordID, contextUser);
                 // The record IS still present and confirmed-unchanged on the source — but skipping
                 // the write here means SetStandardIntegrationFields never runs, so __mj_integration_
                 // LastReconciledAt would freeze at first-sync time. Record the PK so the batch can
@@ -2905,7 +3059,7 @@ export class IntegrationEngine extends BaseSingleton {
         const loaded = await entity.InnerLoad(this.BuildEntityPrimaryKey(record.MatchedMJRecordID, pkFields));
         if (!loaded) {
             // Matched-ID row vanished — fall back to upsert by PK (insert; or update/skip if PK exists)
-            const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser);
+            const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps);
             if (outcome === 'updated')
                 result.RecordsUpdated++;
             else if (outcome === 'skipped')
@@ -2914,21 +3068,34 @@ export class IntegrationEngine extends BaseSingleton {
                 result.RecordsCreated++;
             return;
         }
+        // Set only the BUSINESS fields first, and evaluate dirtiness BEFORE the integration
+        // bookkeeping columns are stamped. This ordering is the whole point: several
+        // __mj_integration_* columns (LastSyncedAt, LastReconciledAt) are set to `new Date()`
+        // on every record, so stamping them first made `entity.Dirty` unconditionally true and
+        // the check below dead — every matched record rewrote every column on every sync,
+        // clobbering concurrent edits to untouched columns and fabricating a record-change row
+        // each time. CreateRecord's upsert branch already orders it this way; this is the same
+        // footprint-clean rule applied to the matched path.
         this.SetEntityFields(entity, record.MappedFields);
-        this.SetStandardIntegrationFields(entity, record);
-        // Skip unchanged records — if no field values actually changed after setting,
-        // don't write to DB. Uses MJ's built-in dirty tracking (zero custom comparison logic).
-        // Critical for connectors without server-side date filtering (e.g., YM) where every
-        // sync re-fetches all records. Without this, 50k+ records get re-written every run.
-        if (!entity.Dirty) {
+        // Skip unchanged records — if no business field values actually changed, don't write.
+        // Uses MJ's built-in dirty tracking (zero custom comparison logic). Critical for
+        // connectors without server-side date filtering (e.g., YM) where every sync re-fetches
+        // all records. Without this, 50k+ records get re-written every run.
+        if (!entity.Dirty && !this.needsSyncStateRepair(entity, entityInfo)) {
             result.RecordsSkipped++;
             // Re-establish the record map even when the write is skipped — see the content-hash skip
             // above for the full rationale (a key-field/PK match can land here with no map row, and
             // dropping the map silently breaks the 1:1 completeness invariant + orphan detection).
             // The entity is loaded here, so use its actual PK as the EntityRecordID. Idempotent upsert.
-            await this.SaveRecordMap(companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|'), contextUser);
+            await this.QueueRecordMap(recordMaps, companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|'), contextUser);
+            // Same reasoning as the content-hash skip: the record IS confirmed present on the
+            // source, so let the batch refresh LastReconciledAt in one set-based touch rather
+            // than rewriting the row for a timestamp.
+            if (reconciledSkipIds)
+                reconciledSkipIds.push(record.MatchedMJRecordID);
             return;
         }
+        this.SetStandardIntegrationFields(entity, record);
         // A5: Pre-write validation
         this.validateEntity(entity, record.MJEntityName);
         const saved = await entity.Save();
@@ -2945,7 +3112,7 @@ export class IntegrationEngine extends BaseSingleton {
         // rows and orphan/delete detection silently degrades. SaveRecordMap is an upsert keyed on
         // (CompanyIntegration, Entity, ExternalID), so this is idempotent for already-mapped records.
         const entityRecordID = entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|');
-        await this.SaveRecordMap(companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entityRecordID, contextUser);
+        await this.QueueRecordMap(recordMaps, companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entityRecordID, contextUser);
         result.RecordsUpdated++;
     }
     /**
@@ -3237,6 +3404,32 @@ export class IntegrationEngine extends BaseSingleton {
     isDateSqlType(t) {
         return /date|time|timestamp/.test(t);
     }
+    /**
+     * True when a row whose BUSINESS fields are unchanged still has to be written because its
+     * integration sync state is stale — it is tombstoned (the record reappeared on the source),
+     * carries a prior error/conflict note, or is not marked Active.
+     *
+     * Without this, restoring UpdateRecord's dirty check would introduce a regression: before
+     * the fix, an unchanged row was rewritten every sync (expensively) and so its tombstone /
+     * error state was cleared as a side effect. Skipping the write must not freeze that state.
+     * The cost is nil — the entity is already loaded at every call site.
+     *
+     * `.Get` on the dynamic `__mj_integration_*` columns is the sanctioned access here: these
+     * runtime-created target tables have no generated entity type, which is why the engine
+     * already `.Set()`s the very same columns in SetStandardIntegrationFields.
+     */
+    needsSyncStateRepair(entity, entityInfo) {
+        if (!entityInfo)
+            return false;
+        const has = (name) => entityInfo.Fields.some(f => f.Name === name);
+        if (has('__mj_integration_IsTombstoned') && entity.Get('__mj_integration_IsTombstoned') === true)
+            return true;
+        if (has('__mj_integration_SyncMessage') && entity.Get('__mj_integration_SyncMessage') != null)
+            return true;
+        if (has('__mj_integration_SyncStatus') && entity.Get('__mj_integration_SyncStatus') !== 'Active')
+            return true;
+        return false;
+    }
     /**
      * Sets standard integration columns (__mj_integration_*) on target entities.
      * Silently skips if the entity doesn't have these columns (e.g., __mj targets).
@@ -3261,12 +3454,17 @@ export class IntegrationEngine extends BaseSingleton {
         if (hasField('__mj_integration_SyncMessage')) {
             entity.Set('__mj_integration_SyncMessage', null);
         }
-        // Content hash of the mapped values — the cheap change-detection key for
+        // Content hash of the MAPPED values only — the cheap change-detection key for
         // watermark-less sources. On the next sync, a record whose freshly-computed
         // hash equals the stored hash can be skipped without loading it (see
         // PrefetchContentHashes / UpdateRecord). No-op on tables predating the column.
+        // Content-hash basis: unmapped/custom keys are EXCLUDED — a newly-appearing custom
+        // column must not break the row match (its capture + stats ride CustomKeyStats;
+        // promotion + the schema-change watermark reset backfill it properly). Rows whose
+        // stored hash predates this basis (overflow folded in) mismatch ONCE, rewrite, and
+        // converge on the new basis.
         if (hasField(CONTENT_HASH_COLUMN)) {
-            entity.Set(CONTENT_HASH_COLUMN, computeContentHashWithOverflow(record.MappedFields ?? {}, record.UnmappedFields));
+            entity.Set(CONTENT_HASH_COLUMN, computeContentHash(record.MappedFields ?? {}));
         }
         // Custom-overflow capture (gaps.md §2): park any source keys with no field map as JSON,
         // in THIS same row write (no extra round-trip → a customs-free sync stays byte-identical).
@@ -3313,6 +3511,94 @@ export class IntegrationEngine extends BaseSingleton {
             entity.Set('__mj_integration_IsTombstoned', false);
         }
     }
+    /**
+     * Reads EVERY record-map row for a (CompanyIntegration, Entity) pair, in pages.
+     *
+     * A plain `RunView` with no `MaxRows` is NOT unbounded: it falls back to the entity's
+     * `UserViewMaxRows`, which defaults to 1000. Both callers of this helper (full-push
+     * "what already exists externally" and full-sync orphan detection) treat the result as
+     * the complete picture, so a silent truncation at 1000 was a data bug in both directions —
+     * duplicate creates on push, and an orphan sweep that cleared only the first 1000 while
+     * reporting a clean run.
+     *
+     * `Complete` is the honest half: paging can still fail part-way, and callers must decide
+     * what a partial read means for them rather than acting on it as if it were whole.
+     *
+     * **Keyset, not OFFSET.** `AfterKey` is used rather than `StartRow` for two independent
+     * reasons, both of which bite precisely at the volumes this helper exists for:
+     *
+     *  1. `StartRow` + `MaxRows` sets the provider's `usingPagination` flag, which forces a
+     *     `SELECT COUNT(*)` over the whole record-map view *for every page*. Paging 100k
+     *     mappings would issue 20 full counts nobody reads — reintroducing, twenty-fold, the
+     *     speculative-count cost removed elsewhere in this change. Keyset queries deliberately
+     *     do not set that flag.
+     *  2. OFFSET paging assumes the row set does not shift underneath the cursor. This table is
+     *     written by the same sync (`RecordMapBatch`), and on Postgres `ID` defaults to a RANDOM
+     *     uuid, so a concurrent insert can land *before* the cursor and shift every later page —
+     *     silently skipping rows while still reporting `Complete: true`. That is exactly the
+     *     truncation this helper exists to prevent, and the orphan sweep would archive live
+     *     records on the strength of it. Keyset seeks on the last ID seen and cannot shift.
+     */
+    async LoadAllRecordMaps(companyIntegrationID, entityID, contextUser) {
+        const PAGE_SIZE = IntegrationEngine.RecordMapPageSize;
+        // Backstop only: at the default page size this is 50M mappings for one
+        // (CompanyIntegration, Entity) pair. It exists so a provider that ignores the seek key
+        // cannot spin forever, not as a real limit anyone should reach.
+        const MAX_PAGES = 5000;
+        const filter = `CompanyIntegrationID='${companyIntegrationID}' AND EntityID='${entityID}'`;
+        const rv = new RunView();
+        const rows = [];
+        let afterID;
+        for (let pageNo = 0; pageNo < MAX_PAGES; pageNo++) {
+            const page = await rv.RunView({
+                EntityName: 'MJ: Company Integration Record Maps',
+                ExtraFilter: filter,
+                Fields: ['ID', 'EntityRecordID', 'ExternalSystemRecordID'],
+                // Keyset requires the sort to be on the PK alone; it is also what makes the seek
+                // below well-defined.
+                OrderBy: 'ID ASC',
+                AfterKey: afterID ? CompositeKey.FromID(afterID) : undefined,
+                IgnoreMaxRows: true, // the entity-level cap is exactly what we're defeating here
+                MaxRows: PAGE_SIZE,
+                ResultType: 'simple',
+                BypassCache: true, // sync decisions must reflect committed record-map state
+            }, contextUser);
+            if (!page.Success) {
+                return { Rows: rows, Complete: false, Error: page.ErrorMessage ?? 'RunView failed' };
+            }
+            rows.push(...page.Results);
+            if (page.Results.length < PAGE_SIZE)
+                return { Rows: rows, Complete: true };
+            afterID = page.Results[page.Results.length - 1].ID;
+        }
+        // Fell out of the loop with full pages still coming: either a genuinely enormous map or a
+        // provider that ignored the seek key. Either way we do NOT have the complete picture, and
+        // saying so is the whole point of the Complete flag.
+        return {
+            Rows: rows,
+            Complete: false,
+            Error: `Record-map paging exceeded ${MAX_PAGES} pages (${rows.length} rows read) without reaching the end.`,
+        };
+    }
+    /**
+     * Records an external↔MJ mapping — batched when the apply pass supplied a writer, direct
+     * otherwise.
+     *
+     * The batched form exists because the per-record `SaveRecordMap`
+     * below costs three round trips, paid for every record the sync touches including the ones it
+     * decides not to change. The `recordMaps` parameter is optional so that callers outside the
+     * batched apply pass (and every existing test) keep the original immediate-write behaviour.
+     */
+    async QueueRecordMap(recordMaps, companyIntegrationID, externalID, entityID, entityRecordID, contextUser) {
+        if (recordMaps) {
+            // Queue only — the apply loop flushes after the batch transaction commits. Writing here
+            // (or auto-flushing on a full chunk) would put map rows inside a transaction that
+            // Discard() can no longer take back on rollback.
+            recordMaps.Queue({ EntityID: entityID, ExternalID: externalID, EntityRecordID: entityRecordID });
+            return;
+        }
+        await this.SaveRecordMap(companyIntegrationID, externalID, entityID, entityRecordID, contextUser);
+    }
     /**
      * Creates or updates a CompanyIntegrationRecordMap entry to track the external↔MJ mapping.
      */
@@ -3323,10 +3609,16 @@ export class IntegrationEngine extends BaseSingleton {
         // The prior always-NewRecord() behavior created a duplicate map row whenever a
         // record fell through to this path again (e.g. matching missed), which then made
         // every by-external-ID lookup ambiguous. Look up an existing mapping first.
+        // Quoted the same way the batched read quotes it — this is `RecordMapBatch`'s fallback path,
+        // so a value the batch would have found here must be found here too, or the fallback
+        // re-creates the very duplicate the upsert exists to prevent.
+        const quotedExternalID = md instanceof DatabaseProviderBase
+            ? quoteTextLiteral(externalID, md.Dialect)
+            : `'${externalID.replace(/'/g, "''")}'`;
         const rv = new RunView();
         const existing = await rv.RunView({
             EntityName: 'MJ: Company Integration Record Maps',
-            ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND EntityID='${entityID}' AND ExternalSystemRecordID='${externalID.replace(/'/g, "''")}'`,
+            ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND EntityID='${entityID}' AND ExternalSystemRecordID=${quotedExternalID}`,
             Fields: ['ID'],
             MaxRows: 1,
             ResultType: 'simple',
@@ -3402,6 +3694,35 @@ export class IntegrationEngine extends BaseSingleton {
         if (mapResult.RecordsErrored > 0) {
             aggregate.Success = false;
         }
+        // Merge custom-key stats (out-of-band candidates) by entity name. Two maps
+        // targeting the SAME entity merge per-key: occurrences/totals sum, max length wins,
+        // samples concat under the same bounded cap.
+        if (mapResult.CustomKeyStats) {
+            aggregate.CustomKeyStats ??= {};
+            for (const [entityName, stats] of Object.entries(mapResult.CustomKeyStats)) {
+                const existing = aggregate.CustomKeyStats[entityName];
+                if (!existing) {
+                    aggregate.CustomKeyStats[entityName] = stats;
+                    continue;
+                }
+                const byKey = new Map(existing.map(s => [s.Key, s]));
+                for (const s of stats) {
+                    const prior = byKey.get(s.Key);
+                    if (!prior) {
+                        byKey.set(s.Key, s);
+                    }
+                    else {
+                        prior.Occurrences += s.Occurrences;
+                        prior.TotalRecords += s.TotalRecords;
+                        prior.MaxLength = Math.max(prior.MaxLength, s.MaxLength);
+                        const room = 20 - prior.SampleValues.length;
+                        if (room > 0)
+                            prior.SampleValues.push(...s.SampleValues.slice(0, room));
+                    }
+                }
+                aggregate.CustomKeyStats[entityName] = [...byKey.values()].sort((a, b) => a.Key.localeCompare(b.Key));
+            }
+        }
     }
     /**
      * Builds a per-entity-map result summary from the entity map and its sync result.
@@ -3648,6 +3969,10 @@ export class IntegrationEngine extends BaseSingleton {
                 ContextUser: contextUser,
                 SyncedEntityNames: syncedEntityNames,
                 Provider: this._provider,
+                // The run's in-memory custom-key candidates — needed because the
+                // overflow-column scan alone under-reports once the hash basis excludes
+                // overflow (skipped rows never write their overflow JSON).
+                CustomKeyStats: result.CustomKeyStats,
             });
         }
         catch (promoteErr) {