@memberjunction/integration-engine 5.27.0 → 5.28.0

package/dist/IntegrationEngine.js

@@ -1,5 +1,5 @@
 import { CompositeKey, Metadata, RunView } from '@memberjunction/core';
-import { BaseSingleton } from '@memberjunction/global';
+import { BaseSingleton, UUIDsEqual } from '@memberjunction/global';
 import { IntegrationEngineBase } from '@memberjunction/integration-engine-base';
 import { ClassifyError } from './types.js';
 import { ConnectorFactory } from './ConnectorFactory.js';
@@ -54,6 +54,79 @@ export class IntegrationEngine extends BaseSingleton {
     static GetAllSyncProgress() {
         return new Map(IntegrationEngine._syncProgress);
     }
+    /**
+     * Resumes any syncs that were orphaned by a process restart.
+     * Finds all CompanyIntegrationRun records with Status='In Progress',
+     * determines which entity maps already completed (have run details),
+     * and resumes from the remaining entity maps using existing watermarks.
+     *
+     * Call this once during MJAPI startup after metadata is loaded.
+     */
+    async ResumeOrphanedSyncs(contextUser) {
+        await IntegrationEngineBase.Instance.Config(false, contextUser);
+        const rv = new RunView();
+        const orphanedRuns = await rv.RunView({
+            EntityName: 'MJ: Company Integration Runs',
+            ExtraFilter: `Status='In Progress'`,
+            ResultType: 'entity_object',
+        }, contextUser);
+        if (!orphanedRuns.Success || orphanedRuns.Results.length === 0) {
+            console.log('[IntegrationEngine] No orphaned syncs to resume');
+            return;
+        }
+        console.log(`[IntegrationEngine] Found ${orphanedRuns.Results.length} orphaned sync(s) to resume`);
+        for (const run of orphanedRuns.Results) {
+            const companyIntegrationID = run.CompanyIntegrationID;
+            const runID = run.ID;
+            try {
+                // Find which entity maps already completed in this run
+                const detailsResult = await rv.RunView({
+                    EntityName: 'MJ: Company Integration Run Details',
+                    ExtraFilter: `CompanyIntegrationRunID='${runID}'`,
+                    Fields: ['EntityID'],
+                    ResultType: 'simple',
+                }, contextUser);
+                const completedEntityIDs = new Set();
+                if (detailsResult.Success) {
+                    for (const d of detailsResult.Results) {
+                        completedEntityIDs.add(d.EntityID.toLowerCase());
+                    }
+                }
+                console.log(`[IntegrationEngine] Resuming run ${runID.substring(0, 8)}... ` +
+                    `for ${companyIntegrationID.substring(0, 8)}... ` +
+                    `(${completedEntityIDs.size} entity maps already completed)`);
+                // Load config and filter to only remaining entity maps
+                const config = await this.LoadRunConfiguration(companyIntegrationID, contextUser);
+                const remainingMaps = config.entityMaps.filter(em => !completedEntityIDs.has((em.EntityID).toLowerCase()));
+                if (remainingMaps.length === 0) {
+                    console.log(`[IntegrationEngine] All entity maps completed for run ${runID.substring(0, 8)}, marking as Success`);
+                    run.EndedAt = new Date();
+                    run.Status = 'Success';
+                    await run.Save();
+                    continue;
+                }
+                console.log(`[IntegrationEngine] Resuming ${remainingMaps.length} remaining entity maps (of ${config.entityMaps.length} total)`);
+                // Replace entityMaps with only the remaining ones
+                config.entityMaps = remainingMaps;
+                // Execute remaining maps using the existing run record
+                const result = await this.ExecuteEntityMaps(config, run, contextUser);
+                result.RunID = runID;
+                await this.FinalizeRun(run, result, contextUser);
+                console.log(`[IntegrationEngine] Resume complete for ${runID.substring(0, 8)}: ` +
+                    `${result.RecordsCreated} created, ${result.RecordsUpdated} updated, ` +
+                    `${result.RecordsErrored} errored`);
+            }
+            catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                console.error(`[IntegrationEngine] Failed to resume run ${runID.substring(0, 8)}: ${errMsg}`);
+                // Mark as failed so it doesn't get picked up again
+                run.EndedAt = new Date();
+                run.Status = 'Failed';
+                run.ErrorLog = JSON.stringify([{ ErrorMessage: `Resume failed: ${errMsg}` }]);
+                await run.Save();
+            }
+        }
+    }
     /**
      * Executes a full sync run for a company integration.
      * If a sync is already running for the same integration, waits for it to finish
@@ -118,12 +191,14 @@ export class IntegrationEngine extends BaseSingleton {
         const run = await this.CreateRunRecord(config.companyIntegration, triggerType, contextUser, options?.ScheduledJobRunID);
         try {
             const result = await this.ExecuteEntityMaps(config, run, contextUser, onProgress, abortSignal);
-            result.RunID = run.Get('ID');
+            result.RunID = run.ID;
             result.Duration = Date.now() - startTime;
             if (result.RecordsErrored > 0) {
                 result.ErrorMessage = `Sync completed with ${result.RecordsErrored} error(s)`;
             }
             await this.FinalizeRun(run, result, contextUser, onNotification);
+            const summary = this.buildSyncResultBody(config.companyIntegration.Integration, result);
+            console.log(`[IntegrationEngine] Sync complete:\n${summary}`);
             return result;
         }
         catch (err) {
@@ -160,7 +235,7 @@ export class IntegrationEngine extends BaseSingleton {
             throw new Error(`CompanyIntegration not found: ${companyIntegrationID}`);
         }
         const integration = integrationsResult.Results
-            .find(i => i.Get('ID') === companyIntegration.IntegrationID);
+            .find(i => UUIDsEqual(i.ID, companyIntegration.IntegrationID));
         if (!integration) {
             throw new Error(`Integration not found for CompanyIntegration: ${companyIntegrationID}`);
         }
@@ -169,7 +244,7 @@ export class IntegrationEngine extends BaseSingleton {
         // Filter to specific entity maps if requested
         if (options?.EntityMapIDs && options.EntityMapIDs.length > 0) {
             const requestedIDs = new Set(options.EntityMapIDs.map(id => id.toLowerCase()));
-            entityMaps = entityMaps.filter(m => requestedIDs.has(m.Get('ID').toLowerCase()));
+            entityMaps = entityMaps.filter(m => requestedIDs.has(m.ID.toLowerCase()));
         }
         return {
             companyIntegration,
@@ -177,6 +252,7 @@ export class IntegrationEngine extends BaseSingleton {
             integration,
             connector,
             fullSync: options?.FullSync ?? false,
+            syncDirection: options?.SyncDirection,
         };
     }
     /**
@@ -186,7 +262,7 @@ export class IntegrationEngine extends BaseSingleton {
         const md = new Metadata();
         const run = await md.GetEntityObject('MJ: Company Integration Runs', contextUser);
         run.NewRecord();
-        run.CompanyIntegrationID = companyIntegration.Get('ID');
+        run.CompanyIntegrationID = companyIntegration.ID;
         run.RunByUserID = contextUser.ID;
         run.StartedAt = new Date();
         run.Status = 'In Progress';
@@ -235,7 +311,7 @@ export class IntegrationEngine extends BaseSingleton {
                 aggregate.EntityMapResults.push(this.buildEntityMapResult(entityMap, mapResult, Date.now() - mapStartTime));
             }
             catch (err) {
-                const objName = entityMap.ExternalObjectName ?? String(entityMap.Get('ID'));
+                const objName = entityMap.ExternalObjectName ?? entityMap.ID;
                 const errMsg = err instanceof Error ? err.message : String(err);
                 console.error(`[IntegrationEngine] Entity map '${objName}' failed: ${errMsg}`);
                 aggregate.RecordsErrored++;
@@ -248,7 +324,7 @@ export class IntegrationEngine extends BaseSingleton {
                     ExternalRecord: { ExternalID: objName, ObjectType: objName, Fields: {} },
                 });
                 aggregate.EntityMapResults.push({
-                    EntityMapID: entityMap.Get('ID'),
+                    EntityMapID: entityMap.ID,
                     ExternalObjectName: objName,
                     EntityName: entityMap.Entity ?? '',
                     Success: false,
@@ -265,18 +341,38 @@ export class IntegrationEngine extends BaseSingleton {
         return aggregate;
     }
     /**
-     * Processes a single entity map: fetch → map → match → validate → apply.
+     * Processes a single entity map based on SyncDirection:
+     * - Pull: fetch from external → map → match → apply to MJ
+     * - Push: detect MJ changes → map → push to external
+     * - Bidirectional: pull first, then push
      */
     async ProcessSingleEntityMap(config, entityMap, run, contextUser, entityMapIndex, totalEntityMaps, onProgress, abortSignal) {
-        const entityMapID = entityMap.Get('ID');
+        const direction = config.syncDirection ?? entityMap.SyncDirection ?? 'Pull';
+        if (direction === 'Pull') {
+            return this.ProcessPullSync(config, entityMap, run, contextUser, entityMapIndex, totalEntityMaps, onProgress, abortSignal);
+        }
+        if (direction === 'Push') {
+            return this.ProcessPushSync(config, entityMap, run, contextUser, entityMapIndex, totalEntityMaps, onProgress, abortSignal);
+        }
+        // Bidirectional: pull first, then push
+        const pullResult = await this.ProcessPullSync(config, entityMap, run, contextUser, entityMapIndex, totalEntityMaps, onProgress, abortSignal);
+        const pushResult = await this.ProcessPushSync(config, entityMap, run, contextUser, entityMapIndex, totalEntityMaps, onProgress, abortSignal);
+        this.MergeResult(pullResult, pushResult);
+        return pullResult;
+    }
+    /**
+     * Pull sync: fetch from external → map → match → validate → apply to MJ.
+     */
+    async ProcessPullSync(config, entityMap, run, contextUser, entityMapIndex, totalEntityMaps, onProgress, abortSignal) {
+        const entityMapID = entityMap.ID;
         const fieldMaps = await this.LoadFieldMaps(entityMapID, contextUser);
-        const watermark = await this.watermarkService.Load(entityMapID, contextUser);
+        const watermark = await this.watermarkService.Load(entityMapID, contextUser, 'Pull');
         // A6: Validate watermark before using it — skip entirely when FullSync requested
         let initialWatermark = config.fullSync ? null : (watermark?.WatermarkValue ?? null);
         if (initialWatermark && watermark) {
             const watermarkType = (watermark.WatermarkType ?? 'Timestamp');
             if (!this.watermarkService.ValidateWatermark(initialWatermark, watermarkType)) {
-                console.warn(`[IntegrationEngine] Invalid watermark '${initialWatermark}' for EntityMap ${entityMap.Get('ID')}, resetting to full fetch`);
+                console.warn(`[IntegrationEngine] Invalid watermark '${initialWatermark}' for EntityMap ${entityMap.ID}, resetting to full fetch`);
                 initialWatermark = null;
             }
         }
@@ -298,16 +394,20 @@ export class IntegrationEngine extends BaseSingleton {
         let currentCursor;
         let batchCount = 0;
         let previousBatchFingerprint;
+        let fetchCompletedCleanly = true; // flipped to false if fetch aborted or errored mid-way
         const MAX_BATCHES_PER_MAP = 5000;
+        const fetchedExternalIDs = new Set(); // Track all IDs seen during this pull for orphan detection
         while (hasMore) {
             if (abortSignal?.aborted) {
                 console.log(`[IntegrationEngine] Sync cancelled for ${entityMap.ExternalObjectName} after ${recordsInMap} records — saving watermark`);
+                fetchCompletedCleanly = false;
                 break;
             }
             batchCount++;
             if (batchCount > MAX_BATCHES_PER_MAP) {
                 console.error(`[IntegrationEngine] Safety limit reached: ${MAX_BATCHES_PER_MAP} batches for ` +
                     `${entityMap.ExternalObjectName}. Stopping to prevent infinite loop.`);
+                fetchCompletedCleanly = false;
                 break;
             }
             const ctx = {
@@ -320,23 +420,35 @@ export class IntegrationEngine extends BaseSingleton {
                 CurrentOffset: currentOffset,
                 CurrentCursor: currentCursor,
             };
-            const batch = await config.connector.FetchChanges(ctx);
-            // Enforce MaxBatchSize — connectors are asked to respect it but may overshoot
+            let batch;
+            try {
+                batch = await config.connector.FetchChanges(ctx);
+            }
+            catch (fetchErr) {
+                const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
+                console.error(`[IntegrationEngine] FetchChanges error for ${entityMap.ExternalObjectName}: ${errMsg}`);
+                fetchCompletedCleanly = false;
+                break;
+            }
+            // If the connector returned more records than MaxBatchSize, log it but never truncate —
+            // all records are written, just in sub-batches to keep DB transactions manageable.
             if (batch.Records.length > this.MaxBatchSize) {
-                console.warn(`[IntegrationEngine] ${entityMap.ExternalObjectName}: connector returned ` +
-                    `${batch.Records.length} records, exceeding MaxBatchSize of ${this.MaxBatchSize}. ` +
-                    `Truncating to ${this.MaxBatchSize}.`);
-                batch.Records = batch.Records.slice(0, this.MaxBatchSize);
+                console.log(`[IntegrationEngine] ${entityMap.ExternalObjectName}: connector returned ` +
+                    `${batch.Records.length} records (> MaxBatchSize ${this.MaxBatchSize}), writing in chunks.`);
             }
             if (batch.Records.length > 0) {
                 const fingerprint = batch.Records.map(r => r.ExternalID).join(',');
                 if (fingerprint === previousBatchFingerprint) {
                     console.warn(`[IntegrationEngine] Duplicate batch detected for ${entityMap.ExternalObjectName} — ` +
                         `connector returned same records twice. Stopping to prevent infinite loop.`);
+                    fetchCompletedCleanly = false;
                     break;
                 }
                 previousBatchFingerprint = fingerprint;
             }
+            for (const rec of batch.Records) {
+                fetchedExternalIDs.add(rec.ExternalID);
+            }
             const mapped = this.fieldMappingEngine.Apply(batch.Records, fieldMaps, entityMap.Entity);
             const resolved = await this.matchEngine.Resolve(mapped, entityMap, fieldMaps, contextUser);
             const beforeApply = result.RecordsCreated + result.RecordsUpdated + result.RecordsSkipped + result.RecordsErrored;
@@ -368,12 +480,332 @@ export class IntegrationEngine extends BaseSingleton {
             hasMore = batch.HasMore === true; // Explicit boolean check — prevents truthy undefined from looping
         }
         if (currentWatermark) {
-            await this.watermarkService.Update(entityMapID, currentWatermark, contextUser);
-            result.WatermarkAfter = currentWatermark;
+            // After a clean full sync, advance the watermark to now rather than the max
+            // modification date found in the last batch. HubSpot's list API returns records
+            // in creation order, so the last batch can contain records with very old
+            // hs_lastmodifieddate values. A stale watermark causes the next incremental to
+            // re-fetch the entire change history from that old date.
+            const finalWatermark = (config.fullSync && fetchCompletedCleanly)
+                ? new Date().toISOString()
+                : currentWatermark;
+            await this.watermarkService.Update(entityMapID, finalWatermark, contextUser, 'Pull');
+            result.WatermarkAfter = finalWatermark;
+        }
+        // Orphan detection: on full sync, delete MJ records whose external counterpart no longer exists.
+        // Only runs if the fetch completed cleanly — a partial fetch (aborted, errored, safety-limited)
+        // means fetchedExternalIDs is incomplete; running deletion on it would be catastrophic.
+        if (config.fullSync && fetchedExternalIDs.size > 0 && fetchCompletedCleanly) {
+            await this.DeleteOrphanedRecords(config.companyIntegration, entityMap, fetchedExternalIDs, result, contextUser);
         }
         await this.CreateRunDetail(run, entityMap, result, contextUser);
         return result;
     }
+    /**
+     * Push sync: detect local MJ record changes → reverse-map fields → push to external system.
+     *
+     * Uses MJ's Record Changes entity to find records modified since the last push watermark.
+     * Filters out changes made by the integration engine itself to prevent echo loops.
+     * For each changed record, calls the connector's CreateRecord/UpdateRecord/DeleteRecord.
+     */
+    async ProcessPushSync(config, entityMap, run, contextUser, _entityMapIndex, _totalEntityMaps, _onProgress, _abortSignal) {
+        const entityMapID = entityMap.ID;
+        const fieldMaps = await this.LoadFieldMaps(entityMapID, contextUser);
+        const pushWatermark = await this.watermarkService.Load(entityMapID, contextUser, 'Push');
+        const lastPushAt = pushWatermark?.WatermarkValue ?? null;
+        // Check connector write capability
+        if (!config.connector.SupportsCreate && !config.connector.SupportsUpdate) {
+            console.log(`[IntegrationEngine] Push skipped for ${entityMap.ExternalObjectName}: connector does not support writes`);
+            return this.EmptyResult();
+        }
+        // Full push: load ALL records from the MJ entity. Incremental push: only changed records.
+        const changedRecords = config.fullSync
+            ? await this.LoadAllMJRecords(entityMap, config.companyIntegration, contextUser)
+            : await this.LoadChangedMJRecords(entityMap, lastPushAt, contextUser);
+        if (changedRecords.length === 0) {
+            console.log(`[IntegrationEngine] Push: no changes for ${entityMap.ExternalObjectName} since ${lastPushAt ?? 'beginning'}`);
+            await this.CreateRunDetail(run, entityMap, this.EmptyResult(), contextUser);
+            return this.EmptyResult();
+        }
+        console.log(`[IntegrationEngine] Push: ${changedRecords.length} changed records for ${entityMap.ExternalObjectName}`);
+        const result = {
+            Success: true, RecordsProcessed: 0, RecordsCreated: 0, RecordsUpdated: 0,
+            RecordsDeleted: 0, RecordsErrored: 0, RecordsSkipped: 0, Errors: [],
+        };
+        // Get push-direction field maps (DestToSource or Both)
+        const pushFieldMaps = fieldMaps.filter(fm => fm.Direction === 'DestToSource' || fm.Direction === 'Both');
+        // If no field maps are configured for push, skip this entity entirely
+        if (pushFieldMaps.length === 0) {
+            console.log(`[IntegrationEngine] Push skipped for ${entityMap.ExternalObjectName}: no field maps with push direction`);
+            return this.EmptyResult();
+        }
+        let latestChangeAt = null;
+        for (const change of changedRecords) {
+            result.RecordsProcessed++;
+            try {
+                await this.PushSingleRecord(change, config, entityMap, pushFieldMaps, result, contextUser);
+                if (change.ChangedAt && (!latestChangeAt || change.ChangedAt > latestChangeAt)) {
+                    latestChangeAt = change.ChangedAt;
+                }
+            }
+            catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                result.RecordsErrored++;
+                result.Errors.push({
+                    ExternalID: change.RecordID,
+                    ChangeType: change.Type,
+                    ErrorMessage: errMsg,
+                    ErrorCode: 'CONNECTOR_ERROR',
+                    Severity: 'Critical',
+                    ExternalRecord: { ExternalID: change.RecordID, ObjectType: entityMap.ExternalObjectName, Fields: {} },
+                });
+            }
+        }
+        // Update push watermark
+        if (latestChangeAt) {
+            await this.watermarkService.Update(entityMapID, latestChangeAt, contextUser, 'Push');
+        }
+        console.log(`[IntegrationEngine] Push complete for ${entityMap.ExternalObjectName}: ` +
+            `${result.RecordsCreated} created, ${result.RecordsUpdated} updated, ` +
+            `${result.RecordsDeleted} deleted, ${result.RecordsErrored} errored`);
+        await this.CreateRunDetail(run, entityMap, result, contextUser);
+        return result;
+    }
+    /** Loads MJ records changed since the last push watermark, excluding integration-engine changes. */
+    async LoadChangedMJRecords(entityMap, lastPushAt, contextUser) {
+        const rv = new RunView();
+        // Query Record Changes for this entity since the last push
+        let filter = `EntityID='${entityMap.EntityID}'`;
+        if (lastPushAt) {
+            filter += ` AND ChangedAt > '${lastPushAt}'`;
+        }
+        // Exclude changes made by the integration engine (System user or specific source)
+        filter += ` AND UserID != 'ECAFCCEC-6A37-EF11-86D4-000D3A4E707E'`; // System user
+        filter += ` AND Type IN ('Create', 'Update', 'Delete')`;
+        const result = await rv.RunView({
+            EntityName: 'Record Changes',
+            ExtraFilter: filter,
+            OrderBy: 'ChangedAt ASC',
+            Fields: ['RecordID', 'Type', 'ChangedAt'],
+            ResultType: 'simple',
+        }, contextUser);
+        if (!result.Success)
+            return [];
+        // Dedupe: keep only the latest change per RecordID.
+        // RecordChange.RecordID format is "fieldName|value" or "f1|v1||f2|v2" for composites.
+        // Normalize to just the value(s) so it matches CompanyIntegrationRecordMap.EntityRecordID.
+        const latestByRecord = new Map();
+        for (const r of result.Results) {
+            const normalizedID = this.NormalizeRecordChangeID(r.RecordID);
+            latestByRecord.set(normalizedID, { ...r, RecordID: normalizedID, Fields: {} });
+        }
+        // Load current field values for each changed record
+        const md = new Metadata();
+        const entityInfo = md.Entities.find(e => e.Name === entityMap.Entity);
+        const pkFieldName = entityInfo?.FirstPrimaryKey?.Name ?? 'ID';
+        for (const [recordID, change] of latestByRecord) {
+            if (change.Type === 'Delete')
+                continue; // No fields to load for deletes
+            try {
+                const entity = await md.GetEntityObject(entityMap.Entity, contextUser);
+                const loaded = await entity.InnerLoad(new CompositeKey([{ FieldName: pkFieldName, Value: recordID }]));
+                if (loaded) {
+                    change.Fields = entity.GetAll();
+                }
+            }
+            catch {
+                // Record may have been deleted between change detection and load
+            }
+        }
+        return [...latestByRecord.values()];
+    }
+    /**
+     * Loads ALL records from an MJ entity for full push sync.
+     * For each record, checks the record map to determine if it's a Create (no external ID)
+     * or Update (has external ID) in the external system.
+     */
+    async LoadAllMJRecords(entityMap, companyIntegration, contextUser) {
+        const rv = new RunView();
+        // Load all records from the MJ entity
+        const allResult = await rv.RunView({
+            EntityName: entityMap.Entity,
+            ResultType: 'simple',
+        }, contextUser);
+        if (!allResult.Success || allResult.Results.length === 0)
+            return [];
+        // Load existing record maps to know which records already exist externally
+        const mapResult = await rv.RunView({
+            EntityName: 'MJ: Company Integration Record Maps',
+            ExtraFilter: `CompanyIntegrationID='${companyIntegration.ID}' AND EntityID='${entityMap.EntityID}'`,
+            Fields: ['EntityRecordID', 'ExternalSystemRecordID'],
+            ResultType: 'simple',
+        }, contextUser);
+        const existingMaps = new Map();
+        if (mapResult.Success) {
+            for (const m of mapResult.Results) {
+                existingMaps.set(m.EntityRecordID, m.ExternalSystemRecordID);
+            }
+        }
+        const md = new Metadata();
+        const entityInfo = md.Entities.find(e => e.Name === entityMap.Entity);
+        const pkFieldName = entityInfo?.FirstPrimaryKey?.Name ?? 'ID';
+        const now = new Date().toISOString();
+        return allResult.Results.map(record => {
+            const recordID = String(record[pkFieldName] ?? '');
+            return {
+                RecordID: recordID,
+                Type: existingMaps.has(recordID) ? 'Update' : 'Create',
+                ChangedAt: now,
+                Fields: record,
+            };
+        });
+    }
+    /** Pushes a single changed MJ record to the external system. */
+    async PushSingleRecord(change, config, entityMap, pushFieldMaps, result, contextUser) {
+        // Reverse-map MJ fields to external fields
+        const externalAttributes = {};
+        for (const fm of pushFieldMaps) {
+            const mjValue = change.Fields[fm.DestinationFieldName];
+            if (mjValue !== undefined && mjValue !== null) {
+                externalAttributes[fm.SourceFieldName] = mjValue;
+            }
+        }
+        // Nothing to push — all field values were null/undefined. Skip instead of erroring.
+        if (change.Type !== 'Delete' && Object.keys(externalAttributes).length === 0) {
+            result.RecordsSkipped++;
+            return;
+        }
+        // Look up the ExternalID from the record map
+        const rv = new RunView();
+        const mapResult = await rv.RunView({
+            EntityName: 'MJ: Company Integration Record Maps',
+            ExtraFilter: `EntityRecordID='${change.RecordID}' AND CompanyIntegrationID='${config.companyIntegration.ID}'`,
+            Fields: ['ExternalSystemRecordID'],
+            MaxRows: 1,
+            ResultType: 'simple',
+        }, contextUser);
+        const externalID = mapResult.Success && mapResult.Results.length > 0
+            ? mapResult.Results[0].ExternalSystemRecordID
+            : null;
+        const crudBase = {
+            CompanyIntegration: config.companyIntegration,
+            ObjectName: entityMap.ExternalObjectName,
+            ContextUser: contextUser,
+        };
+        if (change.Type === 'Delete' && externalID && config.connector.SupportsDelete) {
+            const delResult = await config.connector.DeleteRecord({ ...crudBase, ExternalID: externalID });
+            if (!delResult.Success) {
+                if (delResult.StatusCode === 403) {
+                    console.warn(`[IntegrationEngine] Skipping delete — ${delResult.ErrorMessage}`);
+                    result.RecordsSkipped++;
+                    return;
+                }
+                throw new Error(delResult.ErrorMessage ?? 'Delete failed');
+            }
+            result.RecordsDeleted++;
+        }
+        else if (externalID) {
+            // Update existing external record
+            const updResult = await config.connector.UpdateRecord({
+                ...crudBase, ExternalID: externalID, Attributes: externalAttributes,
+            });
+            if (!updResult.Success) {
+                if (updResult.StatusCode === 403) {
+                    console.warn(`[IntegrationEngine] Skipping update — ${updResult.ErrorMessage}`);
+                    result.RecordsSkipped++;
+                    return;
+                }
+                throw new Error(updResult.ErrorMessage ?? 'Update failed');
+            }
+            result.RecordsUpdated++;
+        }
+        else if (config.connector.SupportsCreate) {
+            // Create new external record
+            const createResult = await config.connector.CreateRecord({
+                ...crudBase, Attributes: externalAttributes,
+            });
+            if (!createResult.Success) {
+                if (createResult.StatusCode === 403) {
+                    console.warn(`[IntegrationEngine] Skipping create — ${createResult.ErrorMessage}`);
+                    result.RecordsSkipped++;
+                    return;
+                }
+                throw new Error(createResult.ErrorMessage ?? 'Create failed');
+            }
+            // Persist the new external ID so future syncs update instead of re-creating
+            if (createResult.ExternalID) {
+                await this.SaveRecordMap(config.companyIntegration.ID, createResult.ExternalID, entityMap.EntityID, change.RecordID, contextUser);
+            }
+            result.RecordsCreated++;
+        }
+        else {
+            result.RecordsSkipped++;
+        }
+    }
+    /**
+     * Full-sync orphan detection: finds MJ records that have a record map entry
+     * but were NOT returned by the external system during this full pull.
+     * These records were deleted externally and should be removed from MJ.
+     */
+    async DeleteOrphanedRecords(companyIntegration, entityMap, fetchedExternalIDs, result, contextUser) {
+        const rv = new RunView();
+        const mapResult = await rv.RunView({
+            EntityName: 'MJ: Company Integration Record Maps',
+            ExtraFilter: `CompanyIntegrationID='${companyIntegration.ID}' ` +
+                `AND EntityID='${entityMap.EntityID}'`,
+            Fields: ['EntityRecordID', 'ExternalSystemRecordID'],
+            ResultType: 'simple',
+        }, contextUser);
+        if (!mapResult.Success)
+            return;
+        const orphans = mapResult.Results.filter(m => !fetchedExternalIDs.has(m.ExternalSystemRecordID));
+        if (orphans.length === 0)
+            return;
+        console.log(`[IntegrationEngine] Orphan detection for ${entityMap.ExternalObjectName}: ${orphans.length} records in MJ not found in external system`);
+        const md = new Metadata();
+        const entityInfo = md.Entities.find(e => e.Name === entityMap.Entity);
+        const pkFields = entityInfo?.PrimaryKeys ?? (entityInfo?.FirstPrimaryKey ? [entityInfo.FirstPrimaryKey] : []);
+        for (const orphan of orphans) {
+            try {
+                const entity = await md.GetEntityObject(entityMap.Entity, contextUser);
+                const loaded = await entity.InnerLoad(this.BuildEntityPrimaryKey(orphan.EntityRecordID, pkFields));
+                if (!loaded) {
+                    console.log(`[IntegrationEngine] Orphan ${orphan.EntityRecordID} already deleted from MJ`);
+                    continue;
+                }
+                const deleted = await entity.Delete();
+                if (deleted) {
+                    result.RecordsDeleted++;
+                    console.log(`[IntegrationEngine] Deleted orphan ${entityMap.Entity} ${orphan.EntityRecordID} (external ${orphan.ExternalSystemRecordID} no longer exists)`);
+                }
+                else {
+                    const reason = entity.LatestResult?.CompleteMessage ?? 'unknown reason';
+                    console.warn(`[IntegrationEngine] Orphan delete blocked for ${entityMap.Entity} ${orphan.EntityRecordID} — ${reason}`);
+                }
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.warn(`[IntegrationEngine] Orphan delete failed for ${entityMap.Entity} ${orphan.EntityRecordID}: ${msg}`);
+            }
+        }
+    }
+    /**
+     * Strips field-name prefixes from a RecordChange.RecordID to get the raw entity record ID.
+     * "hs_object_id|465950833372" → "465950833372"
+     * "contact_id|123||deal_id|456" → "123|456"
+     */
+    NormalizeRecordChangeID(recordID) {
+        return recordID
+            .split('||')
+            .map(part => { const i = part.indexOf('|'); return i >= 0 ? part.substring(i + 1) : part; })
+            .join('|');
+    }
+    /** Returns an empty SyncResult. */
+    EmptyResult() {
+        return {
+            Success: true, RecordsProcessed: 0, RecordsCreated: 0, RecordsUpdated: 0,
+            RecordsDeleted: 0, RecordsErrored: 0, RecordsSkipped: 0, Errors: [],
+        };
+    }
     /**
      * Emits a progress update via the callback.
      */
@@ -436,13 +868,16 @@ export class IntegrationEngine extends BaseSingleton {
                 result.RecordsCreated++;
                 break;
             case 'Update':
-                await this.UpdateRecord(record, contextUser);
-                result.RecordsUpdated++;
+                await this.UpdateRecord(record, companyIntegration, entityMap, result, contextUser);
                 break;
-            case 'Delete':
-                await this.DeleteRecord(record, entityMap, contextUser);
-                result.RecordsDeleted++;
+            case 'Delete': {
+                const didDelete = await this.DeleteRecord(record, entityMap, contextUser);
+                if (didDelete)
+                    result.RecordsDeleted++;
+                else
+                    result.RecordsErrored++;
                 break;
+            }
             case 'Skip':
                 result.RecordsSkipped++;
                 break;
@@ -463,15 +898,25 @@ export class IntegrationEngine extends BaseSingleton {
         if (!saved) {
             throw new Error(`Failed to create ${record.MJEntityName} record`);
         }
-        // Use the external ID as the entity record identifier in the record map
-        await this.SaveRecordMap(companyIntegration.Get('ID'), record.ExternalRecord.ExternalID, entityMap.EntityID, record.ExternalRecord.ExternalID, contextUser);
+        // Use the entity's actual PK (e.g. the UUID assigned by the DB) as the
+        // EntityRecordID in the record map, NOT the external ID. Storing the
+        // external ID as EntityRecordID caused UpdateRecord to fail to load the
+        // entity (UUID lookup with a HubSpot numeric ID) and fall back to
+        // CreateRecord, producing duplicates on every incremental sync.
+        const entityRecordID = entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|');
+        await this.SaveRecordMap(companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entityRecordID, contextUser);
     }
     /**
      * Updates an existing MJ record with pre-write validation.
+     * If the record cannot be loaded (e.g. it was deleted or never fully created),
+     * falls back to CreateRecord (upsert behavior).
      */
-    async UpdateRecord(record, contextUser) {
+    async UpdateRecord(record, companyIntegration, entityMap, result, contextUser) {
         if (!record.MatchedMJRecordID) {
-            throw new Error('Cannot update record without MatchedMJRecordID');
+            // No matched ID — treat as a new record
+            await this.CreateRecord(record, companyIntegration, entityMap, contextUser);
+            result.RecordsCreated++;
+            return;
         }
         const md = new Metadata();
         const entity = await md.GetEntityObject(record.MJEntityName, contextUser);
@@ -479,16 +924,28 @@ export class IntegrationEngine extends BaseSingleton {
         const pkFields = entityInfo?.PrimaryKeys ?? (entityInfo?.FirstPrimaryKey ? [entityInfo.FirstPrimaryKey] : []);
         const loaded = await entity.InnerLoad(this.BuildEntityPrimaryKey(record.MatchedMJRecordID, pkFields));
         if (!loaded) {
-            throw new Error(`Failed to load ${record.MJEntityName} record ${record.MatchedMJRecordID}`);
+            // Record doesn't exist in DB — fall back to INSERT (upsert)
+            await this.CreateRecord(record, companyIntegration, entityMap, contextUser);
+            result.RecordsCreated++;
+            return;
         }
         this.SetEntityFields(entity, record.MappedFields);
         this.SetStandardIntegrationFields(entity, record);
+        // Skip unchanged records — if no field values actually changed after setting,
+        // don't write to DB. Uses MJ's built-in dirty tracking (zero custom comparison logic).
+        // Critical for connectors without server-side date filtering (e.g., YM) where every
+        // sync re-fetches all records. Without this, 50k+ records get re-written every run.
+        if (!entity.Dirty) {
+            result.RecordsSkipped++;
+            return;
+        }
         // A5: Pre-write validation
         this.validateEntity(entity, record.MJEntityName);
         const saved = await entity.Save();
         if (!saved) {
             throw new Error(`Failed to update ${record.MJEntityName} record ${record.MatchedMJRecordID}`);
         }
+        result.RecordsUpdated++;
     }
     /**
      * Runs Validate() on an entity if the method exists.
@@ -508,22 +965,26 @@ export class IntegrationEngine extends BaseSingleton {
      */
     async DeleteRecord(record, entityMap, contextUser) {
         if (!record.MatchedMJRecordID) {
-            throw new Error('Cannot delete record without MatchedMJRecordID');
+            console.log(`[IntegrationEngine] Skipping delete for ${record.ExternalRecord.ObjectType} ExternalID=${record.ExternalRecord.ExternalID} — no MJ record map entry (record was never synced)`);
+            return false;
         }
         if (entityMap.DeleteBehavior === 'DoNothing')
-            return;
+            return false;
         const md = new Metadata();
         const entity = await md.GetEntityObject(record.MJEntityName, contextUser);
         const entityInfo = md.Entities.find(e => e.Name === record.MJEntityName);
         const pkFields = entityInfo?.PrimaryKeys ?? (entityInfo?.FirstPrimaryKey ? [entityInfo.FirstPrimaryKey] : []);
         const loaded = await entity.InnerLoad(this.BuildEntityPrimaryKey(record.MatchedMJRecordID, pkFields));
         if (!loaded) {
-            throw new Error(`Failed to load ${record.MJEntityName} for deletion: ${record.MatchedMJRecordID}`);
+            console.log(`[IntegrationEngine] Skipping delete for ${record.MJEntityName} ${record.MatchedMJRecordID} — record not found in MJ DB (may have been deleted already)`);
+            return false;
         }
         const deleted = await entity.Delete();
         if (!deleted) {
-            throw new Error(`Failed to delete ${record.MJEntityName} record ${record.MatchedMJRecordID}`);
+            const reason = entity.LatestResult?.CompleteMessage ?? 'unknown reason';
+            console.warn(`[IntegrationEngine] Delete blocked for ${record.MJEntityName} ${record.MatchedMJRecordID} — ${reason}`);
         }
+        return deleted;
     }
     /**
      * Builds a CompositeKey for a record lookup using the entity's PK field definitions.
@@ -589,7 +1050,7 @@ export class IntegrationEngine extends BaseSingleton {
         const md = new Metadata();
         const detail = await md.GetEntityObject('MJ: Company Integration Run Details', contextUser);
         detail.NewRecord();
-        detail.CompanyIntegrationRunID = run.Get('ID');
+        detail.CompanyIntegrationRunID = run.ID;
         detail.EntityID = entityMap.EntityID;
         detail.RecordID = `Processed:${result.RecordsProcessed}`;
         detail.Action = result.RecordsCreated > 0 ? 'INSERT' : 'UPDATE';
@@ -616,7 +1077,7 @@ export class IntegrationEngine extends BaseSingleton {
      */
     buildEntityMapResult(entityMap, mapResult, duration) {
         return {
-            EntityMapID: entityMap.Get('ID'),
+            EntityMapID: entityMap.ID,
             ExternalObjectName: entityMap.ExternalObjectName ?? '',
             EntityName: entityMap.Entity ?? '',
             Success: mapResult.RecordsErrored === 0,
@@ -684,7 +1145,7 @@ export class IntegrationEngine extends BaseSingleton {
             Event: event,
             Severity: severity,
             CompanyIntegrationID: run.CompanyIntegrationID,
-            RunID: run.Get('ID'),
+            RunID: run.ID,
             Subject: subject,
             Body: body,
             Result: result,
@@ -708,7 +1169,7 @@ export class IntegrationEngine extends BaseSingleton {
             Event: 'SyncFailed',
             Severity: 'Error',
             CompanyIntegrationID: run.CompanyIntegrationID,
-            RunID: run.Get('ID'),
+            RunID: run.ID,
             Subject: subject,
             Body: body,
             Result: result,