@memberjunction/integration-engine 6.1.0-edge.4 → 6.1.0-edge.6

package/dist/IntegrationEngine.js

@@ -1,5 +1,5 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
-import { CompositeKey, DatabaseProviderBase, LogStatusEx, Metadata, RunView } from '@memberjunction/core';
+import { CompositeKey, DatabaseProviderBase, LogStatusEx, Metadata, RunView, EntitySaveOptions, EntityDeleteOptions } from '@memberjunction/core';
 import { RunOwnershipLostError, RunOwnershipService } from './RunOwnershipService.js';
 import { BaseSingleton, UUIDsEqual } from '@memberjunction/global';
 import { IntegrationEngineBase } from '@memberjunction/integration-engine-base';
@@ -25,6 +25,7 @@ import { AdaptiveConcurrencyController, RunAdaptive } from './AdaptiveConcurrenc
 import { mostRecentWinner } from './ConflictRecency.js';
 import { IntegrationProgressEmitter } from '@memberjunction/integration-progress-artifacts';
 import { CollapseDuplicateIdentities } from './BatchIdentity.js';
+import { WriteSerializer } from './WriteSerializer.js';
 import { ResumeConcurrency, RunResumesBounded } from './ResumeConcurrency.js';
 /** Default batch size for fetching records from external systems */
 const DEFAULT_BATCH_SIZE = 200;
@@ -94,6 +95,33 @@ export class SchemaNotGeneratedError extends Error {
         this.StoredProcedureName = storedProcedureName;
     }
 }
+/**
+ * Thrown by a connector when the vendor says THIS account cannot serve an object it nonetheless
+ * lists in its catalog (e.g. a record type the account has not enabled). Distinct from a fetch
+ * failure: retrying cannot help WITHIN a run, so the map ends cleanly instead of climbing a retry
+ * ladder — one warning, watermark untouched.
+ *
+ * It is deliberately not remembered ACROSS runs. The next run simply asks again, which is what makes
+ * an account change self-healing with no marker, no recheck clock and no override to get right.
+ *
+ * Connectors are NOT required to import this class — the engine also recognises any error carrying
+ * `code === 'OBJECT_UNAVAILABLE'`, so a connector can classify one without a peer version bump.
+ */
+export class ObjectUnavailableError extends Error {
+    constructor(objectName, vendorMessage) {
+        super(`The source cannot serve "${objectName}" for this account: ${vendorMessage}`);
+        this.code = 'OBJECT_UNAVAILABLE';
+        this.name = 'ObjectUnavailableError';
+        this.VendorMessage = vendorMessage;
+    }
+}
+/** Recognises an unavailability signal from a connector that imported the class OR just set the code. */
+export function IsObjectUnavailable(err) {
+    if (err instanceof ObjectUnavailableError)
+        return true;
+    return typeof err === 'object' && err !== null
+        && err.code === 'OBJECT_UNAVAILABLE';
+}
 /**
  * §29 — base of the VALUE-FIT error family: a mapped value cannot fit/coerce into its destination
  * column's type. Per the bounded-typing policy (small columns; large content is an explicit text/json
@@ -149,6 +177,24 @@ const INTEGER_SQL_BOUNDS = {
  * functions are generated, so it must be classified too (else the run produces
  * per-record errors instead of one fail-fast SchemaNotGeneratedError).
  */
+/**
+ * The one primary-key shape a TransactionGroup cannot carry: a SINGLE auto-increment column.
+ *
+ * An enrolled `Save()` returns true immediately and the row lands at `Submit()`, so the caller
+ * reads `entity.PrimaryKey` while the write is still queued. Every other shape survives that —
+ * `NewRecord()` client-generates a `uniqueidentifier`, and a composite/soft key takes its values
+ * from the mapped fields before the save. A server-assigned identity does not exist until the
+ * insert executes, so the record map would be written with a blank EntityRecordID and every
+ * incremental sync would re-insert the row.
+ *
+ * Defined ONCE, module-scope, because it is asked at two different levels — per record at the
+ * enrolment seam, and per entity map before a group is created. Two copies of this rule drifting
+ * apart is how a map ends up batching records that individually refuse to enrol.
+ */
+function IsIdentityOnlyPrimaryKey(primaryKeys) {
+    const pks = primaryKeys ?? [];
+    return pks.length === 1 && pks[0]?.AutoIncrement === true;
+}
 function detectSchemaNotGenerated(entityName, errorMessage) {
     const sqlServer = errorMessage.match(/Could not find stored procedure '([^']+)'/i);
     if (sqlServer)
@@ -176,6 +222,17 @@ function detectSchemaNotGenerated(entityName, errorMessage) {
 export function PositiveInt(v) {
     return typeof v === 'number' && Number.isFinite(v) && v > 0 ? Math.floor(v) : undefined;
 }
+/**
+ * How many deferred writes may accumulate in one batched group before it is submitted and replaced.
+ *
+ * `undefined` (unset, or any unusable value) means no mid-batch flush at all — the batch is one
+ * group and one transaction, which is the default and the behaviour that shipped. An explicit value
+ * trades that atomicity for bounded memory on a heap-constrained box.
+ */
+export function ReadFlushCeiling(env) {
+    const raw = parseInt(env.MJ_INTEGRATION_BATCH_FLUSH_AT ?? '', 10);
+    return Number.isFinite(raw) && raw >= 1 ? raw : undefined;
+}
 export class IntegrationEngine extends BaseSingleton {
     constructor() {
         super();
@@ -240,10 +297,133 @@ export class IntegrationEngine extends BaseSingleton {
      * calls without threading a parameter through ~20 private signatures.
      */
     static { this.runContext = new AsyncLocalStorage(); }
+    /**
+     * The connection's opt-in write mode, from `CompanyIntegration.Configuration`.
+     *
+     * Returns `''` for absent, unparseable or wrongly-typed configuration. Every failure mode
+     * therefore lands on the proven per-record path: a connection has to ASK for batched writes,
+     * and a malformed request is not an ask.
+     */
+    ReadWriteMode(companyIntegration) {
+        try {
+            const raw = companyIntegration.Configuration;
+            if (!raw)
+                return '';
+            const parsed = JSON.parse(raw);
+            return typeof parsed.writeMode === 'string' ? parsed.writeMode : '';
+        }
+        catch {
+            return '';
+        }
+    }
     /** The current run's context, when called from inside a sync run. */
     get currentRunContext() {
         return IntegrationEngine.runContext.getStore();
     }
+    /**
+     * Defers this entity's write into the batch's group, when the connection asked for that.
+     *
+     * A no-op when no group is active, which is every existing caller — an entity constructed
+     * outside a batched apply saves immediately, exactly as before. What this does NOT do is skip
+     * any part of the save: `Save()` still validates, still checks row scope, still renders the
+     * generated procedure call, and still fires its post-save hook. Only the moment the SQL travels
+     * changes, and the group is what makes N of them travel together.
+     */
+    enrolInWriteGroup(entity) {
+        const group = this.currentRunContext?.writeGroup;
+        if (!group)
+            return;
+        // BATCHING REQUIRES A PK THAT EXISTS BEFORE SUBMIT.
+        //
+        // An enrolled `Save()` returns true immediately and the row lands at `Submit()`, so the
+        // caller reads `entity.PrimaryKey` while the write is still queued. That is fine for the
+        // shapes sync actually produces — `NewRecord()` client-generates the UUID for a single
+        // `uniqueidentifier` PK, and a composite/soft PK takes its values from the mapped fields
+        // before the save. It is NOT fine for a single AUTO-INCREMENT PK, whose value only exists
+        // after the insert executes: the record map would be written with a blank EntityRecordID,
+        // which is precisely the "duplicates on every incremental sync" failure the record-map
+        // comment in CreateRecord documents.
+        //
+        // So such an entity is left OUT of the group and saves immediately — correct, one round
+        // trip slower for that entity, and impossible to get silently wrong.
+        if (this.hasIdentityOnlyPrimaryKey(entity))
+            return;
+        entity.TransactionGroup = group;
+    }
+    /**
+     * True when the entity's identity is a SINGLE auto-increment column — the one primary-key
+     * shape whose value cannot be known until the insert has executed. See {@link enrolInWriteGroup}.
+     */
+    hasIdentityOnlyPrimaryKey(entity) {
+        return IsIdentityOnlyPrimaryKey(entity.EntityInfo?.PrimaryKeys);
+    }
+    /**
+     * The MAP-level twin of {@link hasIdentityOnlyPrimaryKey}, asked BEFORE a write group is
+     * created. The entity-level guard refuses enrolment per record, which is total at the seam —
+     * but for a map whose target is that shape it refuses EVERY record, and the branch around it
+     * has already committed to being atomic by construction: it skips `BeginTransaction` because
+     * a group exists. The result is a group that stays empty, no transaction, records saving
+     * one-by-one on their own, and an empty `Submit()` returning true — a NON-ATOMIC batch
+     * reporting success as an atomic one.
+     *
+     * Worse on the failure path: a mid-batch throw correctly skips the rollback (there is no
+     * transaction), then the fallback re-applies the WHOLE batch including rows that already
+     * committed. With a server-assigned identity there is nothing to recognise the first copy
+     * by, so those become duplicates — the very failure the enrolment guard exists to prevent,
+     * reached from the other side.
+     *
+     * So the decision is made here, at the map, and such a map falls to the `useTransaction`
+     * path and gets REAL atomicity instead of an empty group.
+     *
+     * Deliberately not asserted at `Submit()` instead: an empty group is a legitimate state —
+     * when the content-hash fast path skips every record in a batch, nothing enrols and the
+     * group is correctly empty. At submit time the harmful and benign cases are
+     * indistinguishable, which is exactly why this has to be decided up front.
+     */
+    entityMapHasIdentityOnlyPK(entityMap) {
+        if (!entityMap?.Entity)
+            return false;
+        const info = this.ProviderToUse?.EntityByName(entityMap.Entity);
+        return IsIdentityOnlyPrimaryKey(info?.PrimaryKeys);
+    }
+    /**
+     * The connection's write-side-effect mode from `CompanyIntegration.Configuration`. Returns ''
+     * for absent, unparseable or wrongly-typed configuration — every failure mode keeps the side
+     * effects ON. A connection has to ask, and a malformed request is not an ask.
+     */
+    ReadWriteSideEffects(companyIntegration) {
+        try {
+            const raw = companyIntegration.Get('Configuration');
+            if (!raw)
+                return '';
+            const parsed = JSON.parse(raw);
+            return typeof parsed.writeSideEffects === 'string' ? parsed.writeSideEffects : '';
+        }
+        catch {
+            return '';
+        }
+    }
+    /**
+     * Save options for THIS run's machine writes: suppress the audit row + geocode lookup when
+     * the connection asked for it, undefined otherwise (identical to the pre-feature call).
+     * Scoped to the sync's own saves — never to any other writer of the same entities.
+     */
+    get syncSaveOptions() {
+        if (!this.currentRunContext?.suppressWriteSideEffects)
+            return undefined;
+        const opts = new EntitySaveOptions();
+        opts.SkipRecordChanges = true;
+        opts.SkipGeoCoding = true;
+        return opts;
+    }
+    /** Delete twin of {@link syncSaveOptions}. */
+    get syncDeleteOptions() {
+        if (!this.currentRunContext?.suppressWriteSideEffects)
+            return undefined;
+        const opts = new EntityDeleteOptions();
+        opts.SkipRecordChanges = true;
+        return opts;
+    }
     /** Registers (or clears, with undefined) the post-sync custom-column promotion hook. */
     SetPostSyncSchemaPromotionCallback(callback) {
         this.postSyncSchemaPromotionCallback = callback;
@@ -296,19 +476,34 @@ export class IntegrationEngine extends BaseSingleton {
      * phase stays parallel (the real throughput win — it's network-bound). WeakMap so a
      * retired provider's chain entry is collectable.
      */
-    static { this.writeChains = new WeakMap(); }
-    runWriteExclusive(fn) {
+    static { this.writeSerializers = new WeakMap(); }
+    /** The write lock for this engine's provider, created on first use. */
+    writeSerializer() {
         const provider = this.ProviderToUse;
-        let holder = IntegrationEngine.writeChains.get(provider);
-        if (!holder) {
-            holder = { chain: Promise.resolve() };
-            IntegrationEngine.writeChains.set(provider, holder);
-        }
-        // Run fn after the prior write completes (whether it resolved or rejected); keep the chain
-        // alive past failures so one errored batch never deadlocks subsequent writers.
-        const run = holder.chain.then(() => fn(), () => fn());
-        holder.chain = run.then(() => undefined, () => undefined);
-        return run;
+        let s = IntegrationEngine.writeSerializers.get(provider);
+        if (!s) {
+            s = new WriteSerializer();
+            IntegrationEngine.writeSerializers.set(provider, s);
+        }
+        return s;
+    }
+    /**
+     * Runs `fn` with NO other write in flight against the provider. Required for work that opens
+     * the provider's single global transaction — while it is open, any other write would join it.
+     */
+    runWriteExclusive(fn) {
+        return this.writeSerializer().RunExclusive(fn);
+    }
+    /**
+     * Runs `fn` ordered against other writes for the SAME entity map, concurrently with other maps.
+     *
+     * Only for work that opens no provider transaction — watermark bookkeeping, match resolution,
+     * and the post-batch flushes of a batched (TransactionGroup-carrying) apply. Those were queued
+     * behind every other map's writes purely because the lock could not tell them apart from a
+     * transaction-holding section.
+     */
+    runWriteForMap(entityMapID, fn) {
+        return this.writeSerializer().RunKeyed(entityMapID, fn);
     }
     /**
      * Fence check at a batch boundary, BEFORE any write (PR 1 item 3). One SELECT of the run's
@@ -1299,6 +1494,13 @@ export class IntegrationEngine extends BaseSingleton {
      * Processes all entity maps, aggregating results with progress tracking.
      */
     async ExecuteEntityMaps(config, run, contextUser, onProgress, abortSignal, logger) {
+        // Resolve the per-connection side-effect suppression ONCE for the run and stamp it on the
+        // run context, where CreateRecord/UpdateRecord/DeleteRecord (several frames below, no
+        // config parameter) read it back. Both run paths — direct and adopted — come through here.
+        const runCtxForFlags = this.currentRunContext;
+        if (runCtxForFlags) {
+            runCtxForFlags.suppressWriteSideEffects = this.ReadWriteSideEffects(config.companyIntegration) === 'suppressed';
+        }
         const aggregate = {
             Success: true,
             RecordsProcessed: 0,
@@ -1748,6 +1950,107 @@ export class IntegrationEngine extends BaseSingleton {
         });
         await Promise.all(workers);
     }
+    getFetchGate(config) {
+        // OPT-IN: the gate only exists when someone declared a ceiling — a per-connection
+        // `fetchConcurrency` override or the connector's own MaxConcurrencyHint. A connector that
+        // declares neither has never exhibited concurrency-governed throttling, and imposing a
+        // default cap on it would newly constrain fan-outs (lanes × prefetch) that run fine
+        // ungated today. No declaration → no gate → behavior identical to before this feature.
+        const declared = this.getConfigOverrides(config).fetchConcurrency
+            ?? config.connector.MaxConcurrencyHint;
+        if (declared == null)
+            return undefined;
+        this._fetchGates ??= new Map();
+        const key = config.companyIntegration.ID;
+        const ceiling = Math.max(1, declared);
+        let gate = this._fetchGates.get(key);
+        if (!gate || gate.ceiling !== ceiling) {
+            gate = { ceiling, controller: new AdaptiveConcurrencyController({ start: ceiling, min: 1, max: ceiling }), inFlight: 0, waiters: [] };
+            this._fetchGates.set(key, gate);
+        }
+        return gate;
+    }
+    /**
+     * Runs `fn` holding one slot of the connection's fetch gate. A whole retry episode holds a
+     * single slot — retries of a throttled call must not add pressure to the account that just
+     * throttled us. FIFO: waiters resolve in arrival order as slots free or the cap grows.
+     */
+    async withFetchGate(config, fn) {
+        const gate = this.getFetchGate(config);
+        if (!gate)
+            return fn(); // no declared ceiling → ungated, exactly as before
+        while (gate.inFlight >= gate.controller.Cap) {
+            await new Promise((resolve) => gate.waiters.push(resolve));
+        }
+        gate.inFlight++;
+        try {
+            return await fn();
+        }
+        finally {
+            gate.inFlight--;
+            // Wake as many waiters as the CURRENT cap allows — it may have grown (or shrunk)
+            // while they slept; each woken waiter re-checks the cap before taking a slot.
+            while (gate.waiters.length > 0 && gate.inFlight < gate.controller.Cap) {
+                const next = gate.waiters.shift();
+                if (next)
+                    next();
+                else
+                    break;
+            }
+        }
+    }
+    /**
+     * One governed vendor fetch: rate-limit token, adaptive fetch gate, per-attempt timeout,
+     * transient-only retry with Retry-After pacing, and ONE multiplicative decrease per throttle
+     * EPISODE (not per rejected attempt — a 429 that survives three retries is three rejections
+     * but one congestion event, the same distinction TCP draws when it halves the window once per
+     * loss event). Extracted so the loop-top fetch and the pipelined prefetch (which starts the
+     * next page while the current one is processed) share EXACTLY the same pacing and error
+     * semantics — a prefetched page must be indistinguishable from a loop-top fetch to the vendor.
+     *
+     * Retry predicate: our OWN timeout is terminal for this page (WithTimeout is a Promise.race
+     * with no cancellation, so the abandoned attempt keeps running — retrying stacks a second full
+     * page of vendor requests on a source already too slow to finish one); a transport error is
+     * not (a reset socket IS worth retrying). A throttle honors the source's Retry-After via
+     * DelayForError, and every retry re-passes the rate limiter via BeforeRetry so it cannot
+     * bypass a freeze the throttle just applied.
+     */
+    async governedFetch(config, ctx, objectName, fetchTimeoutMs, batchIndex, logger) {
+        let throttleReported = false;
+        try {
+            await this.rateLimit(config);
+            return await this.withFetchGate(config, () => WithRetry(() => WithTimeout(config.connector.FetchChanges(ctx), fetchTimeoutMs, `FetchChanges(${objectName})`), undefined, (err) => !(err instanceof OperationTimeoutError) && IsRetryableError(ClassifyError(err).Code), (attempt, err, delayMs) => {
+                // Report a throttle NOW, not after the retries are spent. ReportThrottle
+                // freezes the shared bucket for this CompanyIntegration, so every other
+                // object fetching concurrently backs off too. Once per episode; later
+                // attempts still get their own Retry-After honoured via DelayForError.
+                if (!throttleReported && ClassifyError(err).Code === 'RATE_LIMIT_EXCEEDED') {
+                    throttleReported = true;
+                    this.reportRateOutcome(config, err);
+                }
+                logger?.emit('sync.fetch.retry', {
+                    externalObjectName: objectName,
+                    batchIndex,
+                    attempt,
+                    delayMs,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }, {
+                DelayForError: (err) => ClassifyError(err).Code === 'RATE_LIMIT_EXCEEDED'
+                    ? config.connector.ExtractRetryAfterMs(err)
+                    : undefined,
+                BeforeRetry: () => this.rateLimit(config),
+            }));
+        }
+        catch (err) {
+            // Retries spent (or none applicable). If the terminal error is itself the throttle
+            // and the retry hook never saw one, apply the episode's one decrease here.
+            if (!throttleReported && ClassifyError(err).Code === 'RATE_LIMIT_EXCEEDED') {
+                this.reportRateOutcome(config, err);
+            }
+            throw err;
+        }
+    }
     /** Minimum ms between outbound requests for this integration (Integration.BatchRequestWaitTime; 0 = disabled). */
     getRequestSpacingMs(config) {
         try {
@@ -1795,6 +2098,8 @@ export class IntegrationEngine extends BaseSingleton {
                 rateLimitBurst: PositiveInt(p.rateLimitBurst),
                 discoveryTimeBudgetMs: PositiveInt(p.discoveryTimeBudgetMs),
                 fetchTimeoutMs: PositiveInt(p.fetchTimeoutMs),
+                // Ceiling on SIMULTANEOUS vendor fetches for this connection (see getFetchGate).
+                fetchConcurrency: PositiveInt(p.fetchConcurrency),
             };
         }
         catch {
@@ -1832,6 +2137,17 @@ export class IntegrationEngine extends BaseSingleton {
      */
     reportRateOutcome(config, throttledErr) {
         const key = config.companyIntegration.ID;
+        // The adaptive fetch gate learns from the SAME signal: a clean outcome creeps the
+        // in-flight cap up by 1; a throttle halves it. This is what makes the gate find the
+        // account's real concurrency grant with zero configuration — including throttles the
+        // connector absorbed inside its own retry and surfaced via ctx.RateLimitReport.
+        const gate = this._fetchGates?.get(key);
+        if (gate) {
+            if (throttledErr === undefined)
+                gate.controller.OnSuccess();
+            else
+                gate.controller.OnThrottleOrError();
+        }
         const rl = this._rateLimiters.get(key);
         if (!rl)
             return;
@@ -1874,11 +2190,27 @@ export class IntegrationEngine extends BaseSingleton {
         this.MergeResult(pullResult, pushResult);
         return pullResult;
     }
+    /** Loads the map fresh and writes its Configuration through Save(), like every other engine write. */
+    async SaveEntityMapConfiguration(entityMap, configurationJSON, contextUser) {
+        const md = new Metadata(); // global-provider-ok: entity-map bookkeeping — single-provider context
+        const row = await md.GetEntityObject('MJ: Company Integration Entity Maps', contextUser);
+        if (!(await row.Load(entityMap.ID)))
+            return;
+        row.Configuration = configurationJSON;
+        await this.runWriteExclusive(() => row.Save());
+        // Keep the in-memory map coherent with what was just persisted, so a later decision in this
+        // same run reads the value that is actually stored.
+        entityMap.Configuration = configurationJSON;
+    }
     /**
      * Pull sync: fetch from external → map → match → validate → apply to MJ.
      */
     async ProcessPullSync(config, entityMap, run, contextUser, entityMapIndex, totalEntityMaps, onProgress, abortSignal, logger) {
         const entityMapID = entityMap.ID;
+        // An object this ACCOUNT cannot serve costs a request, an error event and a retry ladder
+        // every run, forever, and says nothing new after the first time. While the marker is fresh
+        // we spend nothing on it; once it ages out the next attempt IS the recheck, so an object
+        // the account later enables heals itself with no operator action.
         const fieldMaps = await this.LoadFieldMaps(entityMapID, contextUser);
         // Field-level exclusions declared by the connector (SourceFieldInfo.SyncDirective
         // -> IntegrationObjectField.Configuration). Resolved once per map, applied to every
@@ -1890,7 +2222,7 @@ export class IntegrationEngine extends BaseSingleton {
                 excludedFields: Array.from(excludedSourceNames).sort(),
             });
         }
-        const watermark = await this.runWriteExclusive(() => this.watermarkService.Load(entityMapID, contextUser, 'Pull'));
+        const watermark = await this.runWriteForMap(entityMapID, () => this.watermarkService.Load(entityMapID, contextUser, 'Pull'));
         logger?.emit('sync.entity-map.start', {
             phase: 'pull-detail',
             externalObjectName: entityMap.ExternalObjectName,
@@ -2021,6 +2353,11 @@ export class IntegrationEngine extends BaseSingleton {
         const fetchTimeoutMs = PositiveInt(this.getConfigOverrides(config).fetchTimeoutMs)
             ?? PositiveInt(config.connector.FetchChangesTimeoutMs)
             ?? DEFAULT_OPERATION_TIMEOUTS.FetchChangesMs;
+        // Pipelined prefetch state: at most ONE page in flight ahead of processing, keyed by the
+        // cursor it was built from so a drifted position (gap-skip/reset) discards it instead of
+        // consuming the wrong page. If the loop exits with a prefetch still in flight, the promise
+        // settles in the background and its result is discarded (its .catch keeps that silent).
+        let prefetchedNext = null;
         while (hasMore) {
             if (abortSignal?.aborted) {
                 console.log(`[IntegrationEngine] Sync cancelled for ${entityMap.ExternalObjectName} after ${recordsInMap} records — saving watermark`);
@@ -2063,69 +2400,20 @@ export class IntegrationEngine extends BaseSingleton {
             });
             let batch;
             const fetchStart = Date.now();
-            // ONE multiplicative decrease per throttle EPISODE, not one per rejected attempt.
-            //
-            // A 429 that survives three retries is three rejections but one congestion event — the
-            // same distinction TCP draws when it halves the window once per loss event rather than
-            // once per lost segment. Decreasing on each attempt compounds: at a 0.5 backoff factor
-            // three attempts take the rate to an eighth, five take it to a thirtieth, so a single
-            // throttled fetch could drive a connector to its floor purely as a function of how
-            // generous its retry budget is. The freeze already covers the interval the source asked
-            // for; the decrease is about the rate AFTER that, and one signal deserves one step.
-            let throttleReported = false;
             try {
-                await this.rateLimit(config);
-                // Resilient fetch: bound each attempt with a timeout (a hung vendor API must not
-                // hold the sync lock forever) and retry only transient errors (network/throttle/DB).
-                // A non-retryable error (auth, 4xx, parse) throws immediately as before.
-                batch = await WithRetry(() => WithTimeout(config.connector.FetchChanges(ctx), fetchTimeoutMs, `FetchChanges(${entityMap.ExternalObjectName})`), undefined, 
-                // OUR OWN timeout is terminal for this page; a transport error is not.
-                //
-                // `WithTimeout` is a `Promise.race` with no cancellation, so the abandoned attempt
-                // keeps running. Retrying meant a second full page of vendor requests overlapping
-                // the first, then a third — up to 3x the load on a source that was already too slow
-                // to finish once, which is a good way to earn a real 429 (and THAT does cut
-                // concurrency). And the retry could not succeed on its merits anyway: the same work
-                // under the same budget exceeds it again.
-                //
-                // Deliberately `instanceof` rather than the classified code. `ClassifyError` folds
-                // `econnreset` in with timeouts under `NETWORK_TIMEOUT`, and a reset socket IS worth
-                // retrying — so excluding the whole code would lose real resilience. Only the error
-                // WithTimeout itself minted is excluded.
-                (err) => !(err instanceof OperationTimeoutError) && IsRetryableError(ClassifyError(err).Code), (attempt, err, delayMs) => {
-                    // Report a throttle NOW, not after the retries are spent. ReportThrottle
-                    // freezes the shared bucket for this CompanyIntegration, so every other
-                    // object fetching concurrently backs off too — reporting it only in the
-                    // catch below meant the rest of the connector kept hammering a source that
-                    // had already said stop.
-                    //
-                    // Once per episode: see `throttleReported` above. Later attempts still get
-                    // their own Retry-After honoured via DelayForError, which is what actually
-                    // paces this loop; what they must not do is halve the rate again.
-                    if (!throttleReported && ClassifyError(err).Code === 'RATE_LIMIT_EXCEEDED') {
-                        throttleReported = true;
-                        this.reportRateOutcome(config, err);
-                    }
-                    logger?.emit('sync.fetch.retry', {
-                        externalObjectName: entityMap.ExternalObjectName,
-                        batchIndex: batchCount,
-                        attempt,
-                        delayMs,
-                        error: err instanceof Error ? err.message : String(err),
-                    });
-                }, {
-                    // Honour the source's own instruction. A 429 usually carries Retry-After;
-                    // blind exponential backoff ignored it and retried early, which is how a
-                    // soft throttle becomes a hard one. Falls back to backoff when the
-                    // connector cannot parse one.
-                    DelayForError: (err) => ClassifyError(err).Code === 'RATE_LIMIT_EXCEEDED'
-                        ? config.connector.ExtractRetryAfterMs(err)
-                        : undefined,
-                    // A retry must pass through the same gate the first attempt did. The token
-                    // was acquired once before WithRetry, so retries previously bypassed the
-                    // limiter entirely — including the freeze the line above just applied.
-                    BeforeRetry: () => this.rateLimit(config),
-                });
+                if (prefetchedNext && prefetchedNext.key === (currentCursor ?? '')) {
+                    // The page already in flight IS this page — consume it. The rate limiter, the
+                    // fetch gate, the timeout/retry envelope, and once-per-episode throttle
+                    // reporting all ran inside governedFetch when the prefetch was launched, so
+                    // consuming it here adds no vendor pressure and loses no error semantics.
+                    const inFlight = prefetchedNext;
+                    prefetchedNext = null;
+                    batch = await inFlight.promise;
+                }
+                else {
+                    prefetchedNext = null; // position drifted (gap-skip/reset) — discard the stale prefetch
+                    batch = await this.governedFetch(config, ctx, entityMap.ExternalObjectName, fetchTimeoutMs, batchCount, logger);
+                }
                 this.reportRateOutcome(config); // clean fetch → ramp the adaptive rate back up
                 fetchGapCount = 0; // clean fetch → reset the consecutive fetch-gap counter
                 // §10: connector type-driven post-processing hook (default no-op) — enforce/normalize
@@ -2133,17 +2421,68 @@ export class IntegrationEngine extends BaseSingleton {
                 if (batch.Records.length > 0) {
                     batch.Records = batch.Records.map(r => config.connector.PostProcessRecord(r));
                 }
+                // Pipelined prefetch (cursor-paged connectors): the next cursor is known the
+                // moment a page arrives, so start downloading page N+1 while page N is mapped and
+                // written — the shorter leg hides under the longer (~20-30% cycle reduction
+                // measured at a ~6s fetch / ~1-2s process split). Cursor mode ONLY: offset/page
+                // modes interact with the gap-skip resume logic and stay serial. Kill switch:
+                // MJ_INTEGRATION_PREFETCH=off.
+                if ((process.env.MJ_INTEGRATION_PREFETCH ?? 'on') !== 'off' && batch.HasMore === true && batch.NextCursor) {
+                    // Built from the ADVANCED position, exactly as the loop-top rebuild does.
+                    // Spreading ctx with only CurrentCursor once left AfterKeyValue (and
+                    // CurrentOffset) stale, so a keyset connector's "next" page re-ran the
+                    // previous seek: page N+1 was page N again, the duplicate-batch fingerprint
+                    // killed the walk, and every keyset object stopped at exactly two server pages.
+                    const nextCtx = {
+                        ...ctx,
+                        CurrentPage: batch.NextPage,
+                        CurrentOffset: batch.NextOffset,
+                        CurrentCursor: batch.NextCursor,
+                        AfterKeyValue: batch.NextAfterKeyValue ?? ctx.AfterKeyValue,
+                    };
+                    const nextPage = this.governedFetch(config, nextCtx, entityMap.ExternalObjectName, fetchTimeoutMs, batchCount + 1, logger);
+                    nextPage.catch(() => { });
+                    prefetchedNext = { key: batch.NextCursor, promise: nextPage };
+                }
             }
             catch (fetchErr) {
                 const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
                 // A throttle (429 / rate-limit) backs the adaptive limiter off (honoring Retry-After);
                 // other errors don't touch the rate. §5 Gap 2: also flag the map result so the per-layer
                 // AIMD controller reduces in-flight concurrency, not just the per-request token bucket.
+                if (IsObjectUnavailable(fetchErr)) {
+                    // Not a failure to retry: the vendor is telling us this account does not serve
+                    // this object. Warn once and end the map cleanly — no retry ladder, no
+                    // FETCH_INCOMPLETE.
+                    //
+                    // Deliberately NOT remembered between runs. Persisting it would buy one probe
+                    // per object per run, and the object count in any real system is small enough
+                    // that this is not worth a stored marker, a recheck clock, and the staleness
+                    // both bring: a remembered skip is wrong from the moment the account changes,
+                    // and every scheme for noticing that is another thing to get right. Re-asking
+                    // every run is self-healing by construction and has no configuration.
+                    //
+                    // But the map fetched NOTHING, so it is NOT a clean fetch, and every consequence
+                    // of "we saw the complete set" must be withheld. Breaking out with the flag still
+                    // true fell through to the clean-fetch branch and:
+                    //   - minted a wall-clock Timestamp watermark for an object that returned zero
+                    //     records. When the account later enables the object, the next incremental
+                    //     filters `modified > <that stamp>` and permanently misses every record that
+                    //     already existed — destroying the self-healing described above. (An
+                    //     incremental over an EXISTING watermark merely rewrote the same value; the
+                    //     damage lands on a full sync and on the first encounter, where no watermark
+                    //     row exists yet and one is created at "now".)
+                    //   - ran orphan detection. An empty fetch is not evidence that MJ's rows are gone.
+                    //   - overwrote the partition rollup snapshot with an empty map, forcing a full
+                    //     re-diff next run.
+                    fetchCompletedCleanly = false;
+                    logger?.warning(entityMap.ExternalObjectName ?? entityMap.ID, 'OBJECT_UNAVAILABLE', `"${entityMap.ExternalObjectName}" is not available to this account; skipping it until the source starts serving it: ${errMsg}`, { externalObjectName: entityMap.ExternalObjectName });
+                    break;
+                }
                 if (ClassifyError(fetchErr).Code === 'RATE_LIMIT_EXCEEDED') {
-                    // Only if the retry hook did not already do it — a fetch that was retried has
-                    // already had its one decrease applied, at the first sign rather than here.
-                    if (!throttleReported)
-                        this.reportRateOutcome(config, fetchErr);
+                    // The adaptive decrease already happened inside governedFetch (once per
+                    // throttle episode); here we only flag the map result so the per-layer AIMD
+                    // controller reduces in-flight concurrency too.
                     result.Throttled = true;
                 }
                 console.error(`[IntegrationEngine] FetchChanges error for ${entityMap.ExternalObjectName}: ${errMsg}`);
@@ -2169,7 +2508,7 @@ export class IntegrationEngine extends BaseSingleton {
                     // value — a crash from here on resumes from the pre-run watermark and re-covers
                     // the gap. Later checkpoints stop writing floors (gate above).
                     if (watermarkFloorSaved !== null) {
-                        await this.runWriteExclusive(() => this.watermarkService.RestoreValue(entityMapID, preRunWatermarkValue, contextUser));
+                        await this.runWriteForMap(entityMapID, () => this.watermarkService.RestoreValue(entityMapID, preRunWatermarkValue, contextUser));
                         watermarkFloorSaved = null;
                     }
                     logger?.warning(entityMap.ExternalObjectName ?? entityMap.ID, 'FETCH_PAGE_SKIPPED', `Persistent fetch error at ${currentOffset != null ? 'offset ' + currentOffset : 'page ' + currentPage} for ` +
@@ -2303,11 +2642,11 @@ export class IntegrationEngine extends BaseSingleton {
             // not begun"). Holding the same write-lock for the read keeps the connection single-owner.
             const resolved = partitionReconcile
                 ? []
-                : await this.runWriteExclusive(() => this.matchEngine.Resolve(mapped, entityMap, fieldMaps, contextUser));
+                : await this.runWriteForMap(entityMap.ID, () => this.matchEngine.Resolve(mapped, entityMap, fieldMaps, contextUser));
             const beforeApply = result.RecordsCreated + result.RecordsUpdated + result.RecordsSkipped + result.RecordsErrored;
             try {
                 if (!partitionReconcile)
-                    await this.ApplyRecords(resolved, config.companyIntegration, entityMap, result, contextUser, logger, this.getSyncConcurrency(config) <= 1);
+                    await this.ApplyRecords(resolved, config.companyIntegration, entityMap, result, contextUser, logger, this.getSyncConcurrency(config) <= 1, this.getSyncConcurrency(config));
             }
             catch (applyErr) {
                 if (applyErr instanceof SchemaNotGeneratedError) {
@@ -2343,7 +2682,7 @@ export class IntegrationEngine extends BaseSingleton {
                     (batch.HasMore ? ` | more batches pending` : ` | batch complete`));
                 // Update progress on the watermark record so the DB reflects live sync state
                 if (batch.HasMore) {
-                    await this.runWriteExclusive(() => this.watermarkService.UpdateProgress(entityMapID, afterApply, contextUser));
+                    await this.runWriteForMap(entityMapID, () => this.watermarkService.UpdateProgress(entityMapID, afterApply, contextUser));
                 }
             }
             recordsInMap += batch.Records.length;
@@ -2377,7 +2716,7 @@ export class IntegrationEngine extends BaseSingleton {
                 // post-loop save below handles graceful early-exits precisely; this covers a SIGKILL
                 // between graceful checkpoints, costing at most ~25 batches of re-fetch on resume.
                 if (isKeysetConnector && currentAfterKey) {
-                    await this.runWriteExclusive(() => this.watermarkService.SaveKeysetPosition(entityMapID, currentAfterKey, contextUser));
+                    await this.runWriteForMap(entityMapID, () => this.watermarkService.SaveKeysetPosition(entityMapID, currentAfterKey, contextUser));
                 }
                 // The WATERMARK twin of the keyset floor above. Without it, a watermark-based
                 // connector had NO durable position at all until the run ended: a SIGKILL / OOM /
@@ -2395,7 +2734,7 @@ export class IntegrationEngine extends BaseSingleton {
                     && currentWatermark && currentWatermark !== initialWatermark
                     && currentWatermark !== watermarkFloorSaved) {
                     const floor = currentWatermark;
-                    await this.runWriteExclusive(() => this.watermarkService.Update(entityMapID, floor, contextUser, 'Pull'));
+                    await this.runWriteForMap(entityMapID, () => this.watermarkService.Update(entityMapID, floor, contextUser, 'Pull'));
                     watermarkFloorSaved = floor;
                 }
             }
@@ -2432,7 +2771,7 @@ export class IntegrationEngine extends BaseSingleton {
             // NOTE: a connector that ALSO returns a monotonic watermark (MonotonicWatermark=true) skips
             // this branch and falls through to SAVE that watermark below, so its next incremental NARROWS
             // (microtime > watermark) instead of re-scanning the whole object every run.
-            await this.runWriteExclusive(() => this.watermarkService.ClearKeysetPosition(entityMapID, contextUser));
+            await this.runWriteForMap(entityMapID, () => this.watermarkService.ClearKeysetPosition(entityMapID, contextUser));
             result.WatermarkAfter = null;
         }
         else if (fetchCompletedCleanly) {
@@ -2473,13 +2812,13 @@ export class IntegrationEngine extends BaseSingleton {
             else {
                 finalWatermark = new Date().toISOString();
             }
-            await this.runWriteExclusive(() => this.watermarkService.Update(entityMapID, finalWatermark, contextUser, 'Pull'));
+            await this.runWriteForMap(entityMapID, () => this.watermarkService.Update(entityMapID, finalWatermark, contextUser, 'Pull'));
             result.WatermarkAfter = finalWatermark;
         }
         else if (isKeysetConnector && currentAfterKey) {
             // The keyset scan stopped early (cancel / fetch error / safety limit). Persist the precise
             // last ordering key so the next run resumes the seek from here instead of restarting.
-            await this.runWriteExclusive(() => this.watermarkService.SaveKeysetPosition(entityMapID, currentAfterKey, contextUser));
+            await this.runWriteForMap(entityMapID, () => this.watermarkService.SaveKeysetPosition(entityMapID, currentAfterKey, contextUser));
             result.WatermarkAfter = currentAfterKey;
         }
         else if (!hadFetchGap && currentWatermark && currentWatermark !== initialWatermark) {
@@ -2499,7 +2838,7 @@ export class IntegrationEngine extends BaseSingleton {
             // deliberately NOT when hadFetchGap — a skipped page leaves a HOLE behind this watermark,
             // which is why that path holds it for a full re-fetch next run.
             const partialWatermark = currentWatermark;
-            await this.runWriteExclusive(() => this.watermarkService.Update(entityMapID, partialWatermark, contextUser, 'Pull'));
+            await this.runWriteForMap(entityMapID, () => this.watermarkService.Update(entityMapID, partialWatermark, contextUser, 'Pull'));
             result.WatermarkAfter = partialWatermark;
         }
         // Orphan detection: delete/tombstone MJ records whose external counterpart no longer exists.
@@ -2750,13 +3089,19 @@ export class IntegrationEngine extends BaseSingleton {
         // Load current field values for each changed record
         const md = this.ProviderToUse;
         const entityInfo = md.EntityByName(entityMap.Entity);
-        const pkFieldName = entityInfo?.FirstPrimaryKey?.Name ?? 'ID';
+        if (!entityInfo) {
+            throw new Error(`Cannot push ${entityMap.Entity}: entity not found in metadata.`);
+        }
+        // The normalized RecordID is the '|'-joined value(s) of EVERY key column — the shape the record
+        // map stores — so it is parsed against all of the entity's PrimaryKeys. Pinning it to the first
+        // key column (or a made-up `ID`) handed a composite key to one column as "v1|v2" and failed the load.
+        const pkFields = entityInfo.PrimaryKeys;
         for (const [recordID, change] of latestByRecord) {
             if (change.Type === 'Delete')
                 continue; // No fields to load for deletes
             try {
                 const entity = await md.GetEntityObject(entityMap.Entity, contextUser);
-                const loaded = await entity.InnerLoad(new CompositeKey([{ FieldName: pkFieldName, Value: recordID }]));
+                const loaded = await entity.InnerLoad(this.BuildEntityPrimaryKey(recordID, pkFields));
                 if (loaded) {
                     change.Fields = entity.GetAll();
                 }
@@ -2812,10 +3157,15 @@ export class IntegrationEngine extends BaseSingleton {
         }
         const md = this.ProviderToUse;
         const entityInfo = md.EntityByName(entityMap.Entity);
-        const pkFieldName = entityInfo?.FirstPrimaryKey?.Name ?? 'ID';
+        if (!entityInfo) {
+            throw new Error(`Cannot push ${entityMap.Entity}: entity not found in metadata.`);
+        }
+        const pkFields = entityInfo.PrimaryKeys;
         const now = new Date().toISOString();
         return allResult.Results.map(record => {
-            const recordID = String(record[pkFieldName] ?? '');
+            // '|'-joined across EVERY key column — the shape EntityRecordID is stored in — so the
+            // existingMaps lookup (Create vs Update) matches a composite key, not just its first column.
+            const recordID = this.ComposeEntityRecordID(record, pkFields);
             return {
                 RecordID: recordID,
                 Type: existingMaps.has(recordID) ? 'Update' : 'Create',
@@ -3066,7 +3416,7 @@ export class IntegrationEngine extends BaseSingleton {
             const md = this.ProviderToUse;
             const entity = await md.GetEntityObject(entityMap.Entity, contextUser);
             const entityInfo = md.EntityByName(entityMap.Entity);
-            const pkFields = entityInfo?.PrimaryKeys ?? (entityInfo?.FirstPrimaryKey ? [entityInfo.FirstPrimaryKey] : []);
+            const pkFields = entityInfo?.PrimaryKeys ?? [];
             const loaded = await entity.InnerLoad(this.BuildEntityPrimaryKey(mjRecordID, pkFields));
             if (!loaded)
                 return;
@@ -3079,7 +3429,7 @@ export class IntegrationEngine extends BaseSingleton {
             }
             // Surface a failed conflict-mark: the engine thinks the row is quarantined, but without the
             // marker the operator has no signal. A silent failure here leaves the record in limbo.
-            const ok = await entity.Save();
+            const ok = await entity.Save(this.syncSaveOptions);
             if (!ok) {
                 logger?.warning(entityMap.ExternalObjectName ?? entityMap.Entity ?? entityMap.ID, 'CONFLICT_MARK_FAILED', `Could not mark MJ record ${mjRecordID} in-conflict: ${entity.LatestResult?.CompleteMessage ?? 'Save() returned false'}`, { mjRecordID, conflictFields });
             }
@@ -3131,7 +3481,7 @@ export class IntegrationEngine extends BaseSingleton {
             `A large or unexpected count can indicate an incomplete upstream fetch, so review before trusting the deletions.`, { orphanCount: orphans.length });
         const md = this.ProviderToUse;
         const entityInfo = md.EntityByName(entityMap.Entity);
-        const pkFields = entityInfo?.PrimaryKeys ?? (entityInfo?.FirstPrimaryKey ? [entityInfo.FirstPrimaryKey] : []);
+        const pkFields = entityInfo?.PrimaryKeys ?? [];
         for (const orphan of orphans) {
             try {
                 const entity = await md.GetEntityObject(entityMap.Entity, contextUser);
@@ -3160,7 +3510,7 @@ export class IntegrationEngine extends BaseSingleton {
                         entity.Set('__mj_integration_IsTombstoned', true);
                     if (hasField('__mj_integration_DeletedDetectedAt'))
                         entity.Set('__mj_integration_DeletedDetectedAt', new Date().toISOString());
-                    const archived = await entity.Save();
+                    const archived = await entity.Save(this.syncSaveOptions);
                     if (archived) {
                         result.RecordsDeleted++;
                         await this.DeleteRecordMapRow(orphan.ID, contextUser);
@@ -3172,7 +3522,7 @@ export class IntegrationEngine extends BaseSingleton {
                     }
                     continue;
                 }
-                const deleted = await entity.Delete();
+                const deleted = await entity.Delete(this.syncDeleteOptions);
                 if (deleted) {
                     result.RecordsDeleted++;
                     await this.DeleteRecordMapRow(orphan.ID, contextUser);
@@ -3338,8 +3688,8 @@ export class IntegrationEngine extends BaseSingleton {
                 // (~line 1644). matchEngine.Resolve reads existing MJ rows on the SHARED provider
                 // connection, so when streams run in parallel (syncConcurrency>1) it must not interleave
                 // with another stream's open write transaction (else "Transaction in progress" / dirty read).
-                const resolved = await this.runWriteExclusive(() => this.matchEngine.Resolve(recs, entityMap, fieldMaps, contextUser));
-                await this.ApplyRecords(resolved, config.companyIntegration, entityMap, result, contextUser, logger, this.getSyncConcurrency(config) <= 1);
+                const resolved = await this.runWriteForMap(entityMap.ID, () => this.matchEngine.Resolve(recs, entityMap, fieldMaps, contextUser));
+                await this.ApplyRecords(resolved, config.companyIntegration, entityMap, result, contextUser, logger, this.getSyncConcurrency(config) <= 1, this.getSyncConcurrency(config));
                 appliedRecords += recs.length;
             }
         }
@@ -3377,7 +3727,14 @@ export class IntegrationEngine extends BaseSingleton {
     // lost batch atomicity is absorbed by the engine's idempotency (upsert-by-identity + content
     // hash) and the safe-floor watermark (advances only on a clean batch). Default true = the
     // proven atomic serial path, unchanged.
-    useTransaction = true) {
+    useTransaction = true, 
+    /**
+     * Requested apply concurrency. Only consulted on the transaction-free path (the one
+     * `useTransaction: false` selects), where records auto-commit independently and can
+     * therefore overlap. Defaults to 1, so a caller that does not pass it keeps the serial
+     * behaviour exactly.
+     */
+    concurrency = 1) {
         // Batched application with per-record failure isolation (the "grace gap" fix).
         // Happy path: each batch of up to APPLY_BATCH_SIZE records commits as a single
         // transaction — small enough to avoid SQL Server lock escalation (~5000 rows) while
@@ -3404,27 +3761,174 @@ export class IntegrationEngine extends BaseSingleton {
             // otherwise load one-by-one. For a watermark-less re-sync where nothing changed,
             // this lets UpdateRecord skip every per-record load. Best-effort: undefined → the
             // existing dirty-flag path runs unchanged.
-            // Serialize the per-batch DB-write across concurrently-synced streams (shared provider
-            // connection ⇒ one transaction at a time). Fetch already happened in parallel upstream;
-            // only this write section is mutually exclusive. A throw inside (e.g. SchemaNotGenerated)
-            // propagates out to fail-stop this entity map, exactly as before.
-            await this.runWriteExclusive(async () => {
+            // WHAT ACTUALLY NEEDS THE MUTEX.
+            //
+            // The shared provider connection holds one transaction at a time, so the section that
+            // OWNS a transaction must be mutually exclusive. `BeginTransaction` + per-record
+            // `Save()` owns one for the whole batch, and stays serialized exactly as before.
+            //
+            // A batched batch does not. A `TransactionGroup` is an in-memory list until `Submit()`:
+            // enrolling an entity validates, checks row scope and renders the CRUD procedure call,
+            // then parks it — no statement travels, no transaction is open. Only `Submit` touches
+            // the connection. Holding the mutex across the whole apply block was therefore
+            // serializing work that never needed it, and that is what made batching and concurrency
+            // mutually exclusive: maps could not overlap on the part where the time actually goes
+            // (fetch, paging, transform, enrolment) because they were queued behind each other's
+            // writes.
+            //
+            // So the batched path takes the mutex only around the writes themselves. One
+            // transaction is still in flight at a time — the invariant is unchanged — but maps
+            // overlap everywhere else, and each keeps its OWN group, so a poison record fails the
+            // map that owns it instead of every map that happened to be batching alongside it.
+            // Deliberately NOT gated on `useTransaction`. That gate is
+            // `getSyncConcurrency(config) <= 1`, so keeping it here would mean batching only ever
+            // happened at concurrency 1 — the exact tradeoff this change exists to remove.
+            // Batching is a property of how the writes TRAVEL; concurrency is a property of how
+            // many maps fetch at once. They are independent.
+            // Batching is a property of the ENTITY MAP, not only of the connection: a map whose
+            // target's whole identity is server-assigned can never enrol a record, so batching it
+            // would produce an empty group and a non-atomic batch reporting success. See
+            // entityMapHasIdentityOnlyPK.
+            const batchedWrites = this.ReadWriteMode(companyIntegration) === 'batched'
+                && !this.entityMapHasIdentityOnlyPK(entityMap);
+            // NEVER nest `runWriteExclusive`: the inner call waits on a chain that already contains
+            // the outer one, which deadlocks. Under the outer mutex the writes are already
+            // serialized, so they run inline; without it they take the mutex individually.
+            const serializeWrite = batchedWrites
+                ? (fn) => this.runWriteForMap(entityMap.ID, fn)
+                : (fn) => fn();
+            const applyOneBatch = async () => {
                 const precheckHashes = await this.PrefetchContentHashes(batch, contextUser);
                 // PKs of records the content-hash fast path skipped this batch — still present and
                 // confirmed-unchanged on the source. Collected so we can refresh LastReconciledAt for
                 // all of them in ONE set-based touch after the batch (instead of a frozen-forever stamp).
                 let reconciledSkipIds = [];
-                if (useTransaction) {
-                    await provider.BeginTransaction();
-                    try {
+                // Batched writes make a batch atomic by construction — the group is one
+                // transaction — so this branch is entered for them regardless of concurrency. At
+                // concurrency > 1 the atomicity is per entity map, and a group failure still
+                // degrades to the record-by-record retry below.
+                if (useTransaction || batchedWrites) {
+                    // Two ways to make this batch atomic, and they differ ONLY in how the writes
+                    // travel. `BeginTransaction` + per-record `Save()` sends one statement per
+                    // record; a TransactionGroup defers each `Save()` to `Submit()`, which sends
+                    // them together. Everything `Save()` does either way — validation, row scope,
+                    // the generated CRUD procedure, Record Changes, `OnAfterSaveExecute` — is
+                    // identical, because the group defers the WRITE and nothing else.
+                    //
+                    // Opt-in per connection, and it fails closed: an absent, unparseable or
+                    // unrecognised `writeMode` keeps the proven path, so the default never changes
+                    // underneath an existing tenant.
+                    const writeGroup = batchedWrites ? await provider.CreateTransactionGroup() : null;
+                    if (writeGroup) {
+                        // The batching itself (MJ#4087). Without this the group is atomic but still
+                        // SERIAL — one round trip per item inside the transaction, which is exactly
+                        // today's behaviour and why this PR was safe to land before the providers
+                        // could honour it. With it, the whole group travels as one statement.
+                        //
+                        // Set here rather than at construction because it is a property of how THIS
+                        // engine wants its batch submitted, not of the provider: any other caller
+                        // creating a group still gets the sequential default.
+                        writeGroup.BatchedSubmit = true;
+                    }
+                    const runCtx = this.currentRunContext;
+                    // A batched batch REQUIRES a run context: the group is handed to
+                    // ApplySingleRecord's frames through a nested scope of it, so without one
+                    // every record would silently save unenrolled and Submit would commit an
+                    // empty group — which returns true, so the batch would report success while
+                    // having quietly run the per-record path. Every sync path enters a run
+                    // context (RunSync and the adopted-run path both wrap their work in one), so
+                    // this cannot happen today; stating it makes that a requirement rather than
+                    // an accident, and turns a future regression into an error instead of a
+                    // silent loss of the feature.
+                    if (writeGroup && !runCtx) {
+                        throw new Error('batched writes were requested but no run context is active — the write group cannot reach the record ' +
+                            'frames, and an unenrolled batch would report success while saving per-record. This is a wiring bug.');
+                    }
+                    if (!writeGroup)
+                        await provider.BeginTransaction();
+                    // EACH BATCH GETS ITS OWN GROUP, IN ITS OWN CONTEXT SCOPE.
+                    //
+                    // Assigning onto the shared run context would be a single slot: the moment two
+                    // maps overlap — which narrowing the mutex now allows — the second would
+                    // overwrite the first's group and enrol its records into the wrong batch.
+                    // Entering a nested AsyncLocalStorage scope instead gives every concurrent
+                    // batch its own `writeGroup`, inherited by the ApplySingleRecord frames below
+                    // that actually construct the entities, and torn down with the scope.
+                    //
+                    // Per-batch groups are also what keeps failures isolated: a poison record fails
+                    // the group its own map owns, and every other map in flight is untouched.
+                    // OPT-IN ceiling on how many writes may sit deferred in one group.
+                    //
+                    // A group holds every enrolled record's rendered SQL and parameters until Submit,
+                    // so peak memory for a batched apply is roughly (maps in flight x group size x row
+                    // size). With wide rows that is the largest allocation a sync makes, and a box that
+                    // has run out of heap has no way to trade a little throughput for headroom.
+                    //
+                    // Unset (the default) means NO mid-batch flush: the batch stays exactly one group
+                    // and one transaction, as today. Setting it below the batch size splits the batch
+                    // into several transactions, which is a real trade — an earlier flush stays
+                    // committed if a later one fails — and is why it is off unless asked for. The
+                    // per-record fallback that follows a failed batch is idempotent, so the split is
+                    // recoverable; it is simply no longer all-or-nothing.
+                    const flushAt = ReadFlushCeiling(process.env);
+                    const runBatch = async () => {
+                        let enrolledSinceFlush = 0;
                         for (const record of batch) {
                             result.RecordsProcessed++;
                             await this.ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps);
+                            if (flushAt === undefined)
+                                continue;
+                            if (++enrolledSinceFlush < flushAt)
+                                continue;
+                            enrolledSinceFlush = 0;
+                            // Mutating the context object is what makes the swap visible: every frame
+                            // below shares this same object through AsyncLocalStorage, so replacing the
+                            // group here is what the next record's enrolment sees.
+                            const ctx = this.currentRunContext;
+                            const full = ctx?.writeGroup;
+                            if (!full)
+                                continue;
+                            const submitted = await serializeWrite(() => full.Submit());
+                            if (!submitted)
+                                throw new Error('Batched write group did not commit');
+                            const fresh = await provider.CreateTransactionGroup();
+                            fresh.BatchedSubmit = true;
+                            ctx.writeGroup = fresh;
+                        }
+                    };
+                    try {
+                        if (writeGroup) {
+                            // Guarded above: a batched batch always has a run context to nest.
+                            await IntegrationEngine.runContext.run({ ...runCtx, writeGroup }, runBatch);
+                        }
+                        else {
+                            await runBatch();
+                        }
+                        if (writeGroup) {
+                            // The group holds every deferred write; Submit is where they land, in
+                            // one transaction. A false return means the group failed as a whole —
+                            // routed into the same catch, so the fallback below is reached by both
+                            // shapes rather than only by a throw.
+                            //
+                            // This is the ONLY part of a batched batch that touches the connection,
+                            // so it is the only part that takes the write mutex.
+                            const submitted = await serializeWrite(() => writeGroup.Submit());
+                            if (!submitted)
+                                throw new Error('Batched write group did not commit');
+                        }
+                        else {
+                            await provider.CommitTransaction();
                         }
-                        await provider.CommitTransaction();
                     }
                     catch (err) {
-                        await provider.RollbackTransaction();
+                        // No shared slot to clear: the group lived in the batch's own context
+                        // scope, which has already unwound.
+                        //
+                        // A group that failed has already rolled itself back and there is no
+                        // provider-level transaction open to roll back — calling it would throw
+                        // over the real error.
+                        if (!batchedWrites)
+                            await provider.RollbackTransaction();
                         // The batch transaction rolled back; the skip-IDs collected during the failed attempt
                         // never committed. Reset and let the per-record retry re-collect only what commits.
                         reconciledSkipIds = [];
@@ -3446,7 +3950,11 @@ export class IntegrationEngine extends BaseSingleton {
                         }
                         // Degrade to per-record application so the failure isolates to the poison
                         // record(s) and every good record in this batch still commits.
-                        await this.applyRecordsIndividually(batch, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps);
+                        // A batched batch runs concurrently with other entity maps, so the
+                        // per-record fallback must NOT open a provider transaction — that state
+                        // is global to the shared provider. Auto-commit instead; see the
+                        // `useProviderTransaction` doc on applyRecordsIndividually.
+                        await this.applyRecordsIndividually(batch, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps, !batchedWrites);
                     }
                 }
                 else {
@@ -3456,7 +3964,23 @@ export class IntegrationEngine extends BaseSingleton {
                     // connection. Per-record error isolation: a poison record is logged + counted; the
                     // rest still commit; the idempotent re-sync + safe-floor watermark reconcile any
                     // partial batch (the atomicity the transactional path provides is not needed here).
-                    for (const record of batch) {
+                    // The records are independent and each auto-commits on its own pooled
+                    // connection, so this is the one place in the apply path where the requested
+                    // concurrency can actually be spent. Running them one at a time made
+                    // syncConcurrency a fetch-only setting: the caller opted into concurrency, paid
+                    // for it by giving up batch atomicity, and then still wrote serially.
+                    //
+                    // A fixed pool of workers pulling from a shared cursor, rather than
+                    // Promise.all over the batch: 500 simultaneous saves would swamp the connection
+                    // pool. The cap is the same knob the fetch side uses, clamped to a sane ceiling.
+                    const applyLimit = Math.max(1, Math.min(16, Math.floor(concurrency) || 1));
+                    let cursor = 0;
+                    // Set by whichever worker sees it; every worker stops at the next pull and the
+                    // error is rethrown after they settle. SchemaNotGeneratedError means the whole
+                    // map cannot proceed, so finishing the remaining records would be wasted work
+                    // against a table that does not exist.
+                    let fatal;
+                    const applyOne = async (record) => {
                         result.RecordsProcessed++;
                         try {
                             // §10 — bounded inline retry for provably-transient save failures (auto-commit per
@@ -3470,7 +3994,8 @@ export class IntegrationEngine extends BaseSingleton {
                         }
                         catch (err) {
                             if (err instanceof SchemaNotGeneratedError) {
-                                throw err;
+                                fatal ??= err;
+                                return;
                             }
                             // §10 — permanent / retry-exhausted → dead-letter (count + log), move on; watermark advances regardless.
                             result.RecordsErrored++;
@@ -3483,20 +4008,40 @@ export class IntegrationEngine extends BaseSingleton {
                                 Severity: classified.Severity,
                             });
                         }
-                    }
+                    };
+                    // allSettled, not all: a worker must never reject, and the counters/dead-letter
+                    // list are only coherent once every worker has stopped touching them.
+                    await Promise.allSettled(Array.from({ length: Math.min(applyLimit, batch.length) }, async () => {
+                        for (;;) {
+                            if (fatal !== undefined)
+                                return;
+                            const next = cursor++;
+                            if (next >= batch.length)
+                                return;
+                            await applyOne(batch[next]);
+                        }
+                    }));
+                    if (fatal !== undefined)
+                        throw fatal;
                 }
                 // After the batch settles (committed, or per-record retried), refresh
                 // LastReconciledAt for every content-hash-skipped row in ONE set-based touch.
                 // Best-effort — a touch failure must never break the sync.
                 if (reconciledSkipIds.length > 0) {
-                    await this.TouchLastReconciledAt(entityMap, reconciledSkipIds, contextUser, logger);
+                    await serializeWrite(() => this.TouchLastReconciledAt(entityMap, reconciledSkipIds, contextUser, logger));
                 }
                 // Write the batch's record maps set-based, now that the records they point at are
                 // committed. Deliberately AFTER the transaction rather than inside it: the mapping
                 // is derived data that the next sync can re-establish by primary key, and keeping
                 // it out of the write transaction keeps that transaction as short as possible.
-                await this.FlushRecordMaps(recordMaps, entityMap, logger);
-            });
+                await serializeWrite(() => this.FlushRecordMaps(recordMaps, entityMap, logger));
+            };
+            // Batched: overlap freely, serializing only the writes above. Otherwise: the whole
+            // block stays under the mutex, because it owns a provider transaction throughout.
+            if (batchedWrites)
+                await applyOneBatch();
+            else
+                await this.runWriteExclusive(applyOneBatch);
         }
     }
     /**
@@ -3579,17 +4124,45 @@ export class IntegrationEngine extends BaseSingleton {
      *
      * Begin/Commit/Rollback are always matched per record (no leaked open transaction).
      */
-    async applyRecordsIndividually(batch, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps) {
+    async applyRecordsIndividually(batch, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps, 
+    // When false, each record is applied WITHOUT a provider transaction (auto-commit).
+    //
+    // The provider holds its transaction state — `_transactionDepth`, the active
+    // `Transaction`, the savepoint counter — as SINGLE FIELDS on the one shared provider
+    // instance. `BeginTransaction` from two concurrently-syncing entity maps therefore
+    // races that counter: the second caller sees depth 2, issues `SAVE TRANSACTION` against
+    // a transaction the first caller may have already committed, and the leaked depth then
+    // fails every subsequent query on the connection with "Transaction has not begun".
+    //
+    // That is safe ONLY while the engine owns the provider exclusively, i.e. sequential
+    // sync (`useTransaction === true`). The batched path is not sequential, and it does not
+    // need a transaction here either: `ApplySingleRecord` performs exactly ONE write
+    // (Create | Update | Delete — record-maps are queued into `RecordMapBatch` and flushed
+    // set-based later), so there is nothing for a transaction to make atomic. A single
+    // statement either commits or it does not, and `WithRetry`'s next attempt starts clean
+    // without a rollback of a transaction that never held anything.
+    //
+    // This mirrors exactly what the concurrent non-batched path already does: it applies
+    // records through the SAME `ApplySingleRecord` with no transaction at all, and has run
+    // at concurrency > 1 in production.
+    useProviderTransaction = true) {
         const provider = this.ProviderToUse;
         for (const record of batch) {
             result.RecordsProcessed++;
             try {
-                // §10 — apply in its own transaction, with bounded inline retry for PROVABLY-TRANSIENT
-                // save failures (NETWORK_TIMEOUT / RATE_LIMIT_EXCEEDED / DATABASE_ERROR per IsRetryableError).
-                // Each attempt rolls back on throw so the next starts clean; a deadlock/momentary timeout
-                // self-heals here. A PERMANENT error (validation/FK/duplicate/config) is NOT retried — it
-                // throws straight out to the dead-letter path below.
+                // §10 — bounded inline retry for PROVABLY-TRANSIENT save failures
+                // (NETWORK_TIMEOUT / RATE_LIMIT_EXCEEDED / DATABASE_ERROR per IsRetryableError).
+                // A PERMANENT error (validation/FK/duplicate/config) is NOT retried — it throws
+                // straight out to the dead-letter path below.
                 await WithRetry(async () => {
+                    if (!useProviderTransaction) {
+                        // Auto-commit: never touches shared provider transaction state, so
+                        // concurrent entity maps cannot corrupt each other. See the parameter doc.
+                        await this.ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps);
+                        return;
+                    }
+                    // Sequential path, unchanged: apply in its own transaction so a deadlock or
+                    // momentary timeout rolls back and the next attempt starts clean.
                     await provider.BeginTransaction();
                     try {
                         await this.ApplySingleRecord(record, companyIntegration, entityMap, result, contextUser, logger, precheckHashes, reconciledSkipIds, recordMaps);
@@ -3663,7 +4236,7 @@ export class IntegrationEngine extends BaseSingleton {
         try {
             switch (record.ChangeType) {
                 case 'Create': {
-                    const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps, logger);
+                    const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps, logger, precheckHashes);
                     if (outcome === 'updated')
                         result.RecordsUpdated++;
                     else if (outcome === 'skipped')
@@ -3735,11 +4308,14 @@ export class IntegrationEngine extends BaseSingleton {
      */
     async CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps, 
     /** Optional — lets the keyless-key guard below surface on the run's event stream. */
-    keylessLogger) {
+    keylessLogger, 
+    /** What the batch already proved about its destination rows, if anything. */
+    precheck) {
         const md = this.ProviderToUse;
         const entity = await md.GetEntityObject(record.MJEntityName, contextUser);
+        this.enrolInWriteGroup(entity);
         const entityInfo = md.EntityByName(record.MJEntityName);
-        const pkFields = entityInfo?.PrimaryKeys ?? (entityInfo?.FirstPrimaryKey ? [entityInfo.FirstPrimaryKey] : []);
+        const pkFields = entityInfo?.PrimaryKeys ?? [];
         // Upsert-safe: if the record's mapped fields carry a PK (soft-PK dest tables key on the external
         // ID), check whether that row already exists before deciding INSERT vs UPDATE. A null mappedPK
         // (e.g. a server-assigned UUID PK not present in the mapped fields) means a genuinely new row.
@@ -3769,7 +4345,16 @@ export class IntegrationEngine extends BaseSingleton {
             console.error(`[IntegrationEngine] KEYLESS_RECORD_REFUSED: ${detail}`);
             return 'skipped';
         }
-        const existed = mappedPK != null
+        // The batch's prefetch may already have proved this row absent. When it did, the load below
+        // is a SELECT * — every column including any NVARCHAR(MAX) — issued once per record, to
+        // learn something the batch established in a single query. Skipping it is the whole point
+        // of widening that prefetch to the create path.
+        //
+        // Only ever used to skip work when absence is PROVEN: the prefetch must have covered every
+        // record in the batch, and this key must be missing from it. Anything less falls through to
+        // the load, because a wrong "absent" turns an update into a duplicate insert.
+        const provablyAbsent = this.isProvablyAbsent(mappedPK, precheck);
+        const existed = mappedPK != null && !provablyAbsent
             ? await entity.InnerLoad(this.BuildEntityPrimaryKey(mappedPK, pkFields))
             : false;
         if (existed) {
@@ -3796,7 +4381,7 @@ export class IntegrationEngine extends BaseSingleton {
             // re-establish the possibly-cleared record map and SKIP the write — leaving __mj_UpdatedAt
             // and the integration LastSynced columns untouched, exactly like the content-hash skip path.
             this.SetEntityFields(entity, record.MappedFields);
-            if (!entity.Dirty && !this.needsSyncStateRepair(entity, entityInfo)) {
+            if (!entity.Dirty && !this.needsSyncStateRepair(entity, entityInfo, record)) {
                 await this.QueueRecordMap(recordMaps, companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|'), contextUser);
                 return 'skipped';
             }
@@ -3808,7 +4393,7 @@ export class IntegrationEngine extends BaseSingleton {
         this.SetStandardIntegrationFields(entity, record);
         // A5: Pre-write validation
         this.validateEntity(entity, record.MJEntityName);
-        const saved = await entity.Save();
+        const saved = await entity.Save(this.syncSaveOptions);
         if (!saved) {
             const errMsg = entity.LatestResult?.CompleteMessage ?? 'unknown error';
             const schemaErr = detectSchemaNotGenerated(record.MJEntityName, errMsg);
@@ -3822,6 +4407,17 @@ export class IntegrationEngine extends BaseSingleton {
         // incremental sync. SaveRecordMap is an upsert keyed on (CompanyIntegration, Entity, ExternalID),
         // so this also re-establishes a map that was previously cleared.
         const entityRecordID = entity.PrimaryKey.KeyValuePairs.map(kv => String(kv.Value)).join('|');
+        // The prefetch's absence proof is only true until this process inserts the row. A mid-batch
+        // flush (MJ_INTEGRATION_BATCH_FLUSH_AT) COMMITS part of a batch; if a later record then fails,
+        // the per-record fallback re-applies the whole batch against the SAME precheck — and the
+        // committed rows' keys, honestly absent at prefetch time, would still "prove" absent and
+        // insert again. Recording the key the moment we create keeps the proof truthful for any
+        // replay in this run. Deliberately unconditional on commit outcome: if the group later rolls
+        // back, an over-included key merely costs that record one existence load on retry — while an
+        // under-included key costs a duplicate row. Only ever err toward the load.
+        if (!existed && mappedPK != null && precheck) {
+            precheck.Present.add(mappedPK);
+        }
         await this.QueueRecordMap(recordMaps, companyIntegration.ID, record.ExternalRecord.ExternalID, entityMap.EntityID, entityRecordID, contextUser);
         return existed ? 'updated' : 'created';
     }
@@ -3849,6 +4445,23 @@ export class IntegrationEngine extends BaseSingleton {
         }
         return values.join('|');
     }
+    /**
+     * True ONLY when the batch prefetch PROVED this record's row does not exist: the prefetch covered
+     * every record in the batch AND this key is missing from the rows it found. Anything less is
+     * "unknown", and unknown must load — a wrong "absent" turns an update into a duplicate INSERT.
+     *
+     * `mappedPK` must be the '|'-joined key {@link extractMappedPrimaryKey} returns — the SAME shape
+     * {@link PrefetchContentHashes} keys `Present` with (`pkNames.map(n => row[n] ?? '').join('|')`).
+     * The first version of this check re-derived a key by indexing that string with PK field names,
+     * which evaluates to `''` for every record — so "provably absent" was unconditionally true and
+     * every upsert of an existing row became a blind duplicate INSERT. Kept as its own method so the
+     * decision is testable against the real extractor's output rather than a re-implementation.
+     */
+    isProvablyAbsent(mappedPK, precheck) {
+        return mappedPK != null
+            && precheck?.CoversWholeBatch === true
+            && !precheck.Present.has(mappedPK);
+    }
     /**
      * Updates an existing MJ record with pre-write validation.
      * If the record cannot be loaded (e.g. it was deleted or never fully created),
@@ -3859,7 +4472,7 @@ export class IntegrationEngine extends BaseSingleton {
     logger) {
         if (!record.MatchedMJRecordID) {
             // No matched ID — upsert by PK (insert; or update/skip if the PK already exists)
-            const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps, logger);
+            const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps, logger, precheckHashes);
             if (outcome === 'updated')
                 result.RecordsUpdated++;
             else if (outcome === 'skipped')
@@ -3874,7 +4487,7 @@ export class IntegrationEngine extends BaseSingleton {
         // unchanged — skip the per-record DB load AND the write. The dirty-flag check
         // below is the fallback for entities without the hash column.
         if (precheckHashes) {
-            const stored = precheckHashes.get(record.MatchedMJRecordID);
+            const stored = precheckHashes.Hashes.get(record.MatchedMJRecordID);
             if (stored && stored === computeContentHash(record.MappedFields ?? {})) {
                 result.RecordsSkipped++;
                 // Re-establish the external↔MJ record map even on the content-hash skip. A record can
@@ -3902,12 +4515,13 @@ export class IntegrationEngine extends BaseSingleton {
         }
         const md = this.ProviderToUse;
         const entity = await md.GetEntityObject(record.MJEntityName, contextUser);
+        this.enrolInWriteGroup(entity);
         const entityInfo = md.EntityByName(record.MJEntityName);
-        const pkFields = entityInfo?.PrimaryKeys ?? (entityInfo?.FirstPrimaryKey ? [entityInfo.FirstPrimaryKey] : []);
+        const pkFields = entityInfo?.PrimaryKeys ?? [];
         const loaded = await entity.InnerLoad(this.BuildEntityPrimaryKey(record.MatchedMJRecordID, pkFields));
         if (!loaded) {
             // Matched-ID row vanished — fall back to upsert by PK (insert; or update/skip if PK exists)
-            const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps, logger);
+            const outcome = await this.CreateRecord(record, companyIntegration, entityMap, contextUser, recordMaps, logger, precheckHashes);
             if (outcome === 'updated')
                 result.RecordsUpdated++;
             else if (outcome === 'skipped')
@@ -3929,7 +4543,7 @@ export class IntegrationEngine extends BaseSingleton {
         // Uses MJ's built-in dirty tracking (zero custom comparison logic). Critical for
         // connectors without server-side date filtering (e.g., YM) where every sync re-fetches
         // all records. Without this, 50k+ records get re-written every run.
-        if (!entity.Dirty && !this.needsSyncStateRepair(entity, entityInfo)) {
+        if (!entity.Dirty && !this.needsSyncStateRepair(entity, entityInfo, record)) {
             result.RecordsSkipped++;
             // Re-establish the record map even when the write is skipped — see the content-hash skip
             // above for the full rationale (a key-field/PK match can land here with no map row, and
@@ -3946,7 +4560,7 @@ export class IntegrationEngine extends BaseSingleton {
         this.SetStandardIntegrationFields(entity, record);
         // A5: Pre-write validation
         this.validateEntity(entity, record.MJEntityName);
-        const saved = await entity.Save();
+        const saved = await entity.Save(this.syncSaveOptions);
         if (!saved) {
             const errMsg = entity.LatestResult?.CompleteMessage ?? 'unknown error';
             const schemaErr = detectSchemaNotGenerated(record.MJEntityName, errMsg);
@@ -3973,19 +4587,39 @@ export class IntegrationEngine extends BaseSingleton {
      *   - the read fails (best-effort — a logging/optimization read must never break a sync).
      */
     async PrefetchContentHashes(batch, contextUser) {
-        const ids = Array.from(new Set(batch.filter(r => r.ChangeType === 'Update' && r.MatchedMJRecordID)
-            .map(r => r.MatchedMJRecordID)));
-        if (ids.length === 0)
-            return undefined;
         const entityName = batch[0].MJEntityName;
         const entityInfo = this.ProviderToUse.EntityByName(entityName);
         if (!entityInfo)
             return undefined;
-        if (!entityInfo.Fields.some(f => f.Name === CONTENT_HASH_COLUMN))
+        if (!entityInfo.Fields?.some(f => f.Name === CONTENT_HASH_COLUMN))
             return undefined;
         const pkFields = entityInfo.PrimaryKeys ?? [];
         if (pkFields.length === 0)
             return undefined;
+        // Matched rows contribute the key the matcher already resolved. Unmatched rows contribute
+        // the key their MAPPED FIELDS carry (soft-PK tables key on the external id), which is the
+        // same key CreateRecord is about to probe for one at a time. Asking for all of them in the
+        // one query we are already issuing is what lets that per-record probe be skipped.
+        const wanted = new Set();
+        let everyRecordCovered = true;
+        for (const r of batch) {
+            if (r.ChangeType === 'Update' && r.MatchedMJRecordID) {
+                wanted.add(r.MatchedMJRecordID);
+                continue;
+            }
+            const mappedPK = this.extractMappedPrimaryKey(r, pkFields);
+            if (mappedPK == null) {
+                // A destination-generated key (identity / server-assigned UUID) cannot be known
+                // before the insert, so this record's existence is genuinely unknowable here.
+                everyRecordCovered = false;
+                continue;
+            }
+            // Already the '|'-joined key in pkFields order — add it as-is (see extractMappedPrimaryKey).
+            wanted.add(mappedPK);
+        }
+        const ids = Array.from(wanted);
+        if (ids.length === 0)
+            return undefined;
         // Map keys must match `record.MatchedMJRecordID`, which is the PK value(s) joined by '|' in
         // PrimaryKeys order (single value for single-PK, "v1|v2" for composite — see MatchEngine).
         const pkNames = pkFields.map(f => f.Name);
@@ -4003,19 +4637,32 @@ export class IntegrationEngine extends BaseSingleton {
                 Fields: [...pkNames, CONTENT_HASH_COLUMN],
                 ExtraFilter: extraFilter,
                 ResultType: 'simple',
+                // A plain RunView is NOT unbounded — it falls back to the entity's UserViewMaxRows
+                // (default 1000). This result is what `CoversWholeBatch` absence proofs are judged
+                // against, and coverage is computed from the REQUEST side, never reconciled with
+                // res.Results.length: a silently truncated response would mark every existing row
+                // beyond the cap "provably absent" and re-INSERT it as a duplicate on every sync.
+                // Today the apply batch (500) sits under the default cap, so nothing fires — but a
+                // 2x margin defended by nothing is not a guard. Same reasoning as baseEngine's own
+                // IgnoreMaxRows use, and this file documents the identical trap on the push side.
+                IgnoreMaxRows: true,
             }, contextUser);
             if (!res.Success)
                 return undefined;
-            const map = new Map();
+            const Hashes = new Map();
+            const Present = new Set();
             for (const row of res.Results) {
                 // Re-key by the same '|'-join the matcher produced, so the lookup in ApplySingleRecord hits.
                 const key = pkNames.map(n => row[n] ?? '').join('|');
+                // Presence is recorded for EVERY returned row. A row whose hash is NULL still exists,
+                // and treating "no hash" as "no row" would turn an update into a duplicate insert.
+                Present.add(key);
                 const hash = row[CONTENT_HASH_COLUMN];
                 if (typeof hash === 'string' && hash.length > 0) {
-                    map.set(key, hash);
+                    Hashes.set(key, hash);
                 }
             }
-            return map;
+            return { Hashes, Present, CoversWholeBatch: everyRecordCovered };
         }
         catch (err) {
             // MJ#3047 lesson: this best-effort catch was SILENT, so a failing prefetch (e.g. a reserved-word
@@ -4051,8 +4698,9 @@ export class IntegrationEngine extends BaseSingleton {
             return false;
         const md = this.ProviderToUse;
         const entity = await md.GetEntityObject(record.MJEntityName, contextUser);
+        this.enrolInWriteGroup(entity);
         const entityInfo = md.EntityByName(record.MJEntityName);
-        const pkFields = entityInfo?.PrimaryKeys ?? (entityInfo?.FirstPrimaryKey ? [entityInfo.FirstPrimaryKey] : []);
+        const pkFields = entityInfo?.PrimaryKeys ?? [];
         const loaded = await entity.InnerLoad(this.BuildEntityPrimaryKey(record.MatchedMJRecordID, pkFields));
         if (!loaded) {
             console.log(`[IntegrationEngine] Skipping delete for ${record.MJEntityName} ${record.MatchedMJRecordID} — record not found in MJ DB (may have been deleted already)`);
@@ -4074,14 +4722,14 @@ export class IntegrationEngine extends BaseSingleton {
                 entity.Set('__mj_integration_IsTombstoned', true);
             if (hasField('__mj_integration_DeletedDetectedAt'))
                 entity.Set('__mj_integration_DeletedDetectedAt', new Date().toISOString());
-            const archived = await entity.Save();
+            const archived = await entity.Save(this.syncSaveOptions);
             if (!archived) {
                 const reason = entity.LatestResult?.CompleteMessage ?? 'unknown reason';
                 console.warn(`[IntegrationEngine] Soft-delete (archive) failed for ${record.MJEntityName} ${record.MatchedMJRecordID} — ${reason}`);
             }
             return archived;
         }
-        const deleted = await entity.Delete();
+        const deleted = await entity.Delete(this.syncDeleteOptions);
         if (!deleted) {
             const reason = entity.LatestResult?.CompleteMessage ?? 'unknown reason';
             console.warn(`[IntegrationEngine] Delete blocked for ${record.MJEntityName} ${record.MatchedMJRecordID} — ${reason}`);
@@ -4096,8 +4744,13 @@ export class IntegrationEngine extends BaseSingleton {
      */
     BuildEntityPrimaryKey(recordID, pkFields) {
         const key = new CompositeKey();
-        if (pkFields.length <= 1) {
-            key.KeyValuePairs.push({ FieldName: pkFields[0]?.Name ?? 'ID', Value: recordID });
+        if (pkFields.length === 0) {
+            // Never invent an `ID` column: MJ keys can have any name, and a load against a made-up
+            // field fails with "Primary key ID not found in entity ..." — surface the real cause instead.
+            throw new Error(`Cannot build a primary key for record '${recordID}': the entity has no primary key fields in metadata.`);
+        }
+        if (pkFields.length === 1) {
+            key.KeyValuePairs.push({ FieldName: pkFields[0].Name, Value: recordID });
         }
         else {
             const parts = recordID.split('|');
@@ -4107,6 +4760,14 @@ export class IntegrationEngine extends BaseSingleton {
         }
         return key;
     }
+    /**
+     * The exact inverse of {@link BuildEntityPrimaryKey}: the '|'-joined value(s) of a data row's
+     * primary-key column(s), in PK-field order — the shape CompanyIntegrationRecordMap.EntityRecordID
+     * and a normalized RecordChange.RecordID carry. A single-column key is just its value.
+     */
+    ComposeEntityRecordID(row, pkFields) {
+        return pkFields.map(pk => serializeKeyValue(row[pk.Name])).join('|');
+    }
     /**
      * Reports ACTIVE field maps whose MJ column does not exist, once per entity map per run.
      *
@@ -4291,7 +4952,7 @@ export class IntegrationEngine extends BaseSingleton {
      * runtime-created target tables have no generated entity type, which is why the engine
      * already `.Set()`s the very same columns in SetStandardIntegrationFields.
      */
-    needsSyncStateRepair(entity, entityInfo) {
+    needsSyncStateRepair(entity, entityInfo, record) {
         if (!entityInfo)
             return false;
         const has = (name) => entityInfo.Fields.some(f => f.Name === name);
@@ -4301,6 +4962,27 @@ export class IntegrationEngine extends BaseSingleton {
             return true;
         if (has('__mj_integration_SyncStatus') && entity.Get('__mj_integration_SyncStatus') !== 'Active')
             return true;
+        // A STALE CONTENT HASH is repair-worthy for the same reason: skipping the write freezes it.
+        //
+        // The case that produces one is a source that stops sending a column. The mapper OMITS an
+        // absent key rather than mapping it to null (a missing value is not a null value), so the
+        // recomputed hash differs — but SetEntityFields never touches that column either, so the
+        // entity is NOT dirty and the skip above fires. The stored hash is therefore never refreshed
+        // and the mismatch is permanent: that row loses the content-hash fast path FOREVER, paying a
+        // full load and field-by-field compare on every sync until some other field happens to
+        // change. One repair write here re-converges it, and every later sync skips it cheaply.
+        //
+        // Deliberately NOT treated as "the column is gone" — absence in the data is not evidence of
+        // absence in the schema (§ the same rule the field-level deactivation follows). The value is
+        // left exactly as it is; only the hash is brought back in line with what we are actually
+        // mapping.
+        if (record && has(CONTENT_HASH_COLUMN)) {
+            const storedHash = entity.Get(CONTENT_HASH_COLUMN);
+            if (typeof storedHash === 'string' && storedHash.length > 0
+                && storedHash !== computeContentHash(record.MappedFields ?? {})) {
+                return true;
+            }
+        }
         return false;
     }
     /**
@@ -4731,7 +5413,7 @@ export class IntegrationEngine extends BaseSingleton {
             const d = provider.Dialect;
             const runTable = `${d.QuoteIdentifier(runInfo.SchemaName)}.${d.QuoteIdentifier(runInfo.BaseTable)}`;
             const detailTable = `${d.QuoteIdentifier(detailInfo.SchemaName)}.${d.QuoteIdentifier(detailInfo.BaseTable)}`;
-            const runPk = d.QuoteIdentifier(runInfo.PrimaryKeys[0].Name);
+            const runPk = d.QuoteIdentifier(runInfo.FirstPrimaryKey.Name); // first-pk-ok: runInfo is MJ: Company Integration Runs (core entity, single ID key), the FK target of Run Details.CompanyIntegrationRunID
             const ciCol = d.QuoteIdentifier('CompanyIntegrationID');
             const startedCol = d.QuoteIdentifier('StartedAt');
             const detailFk = d.QuoteIdentifier('CompanyIntegrationRunID');