@memberjunction/integration-engine 6.1.0-edge.2 → 6.1.0-edge.3

package/dist/IntegrationEngine.js

@@ -1,9 +1,11 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
 import { CompositeKey, DatabaseProviderBase, LogStatusEx, Metadata, RunView } from '@memberjunction/core';
+import { RunOwnershipLostError, RunOwnershipService } from './RunOwnershipService.js';
 import { BaseSingleton, UUIDsEqual } from '@memberjunction/global';
 import { IntegrationEngineBase } from '@memberjunction/integration-engine-base';
 import { ClassifyError, IsRetryableError } from './types.js';
 import { WithRetry } from './RetryRunner.js';
-import { WithTimeout, DEFAULT_OPERATION_TIMEOUTS } from './BaseIntegrationConnector.js';
+import { WithTimeout, OperationTimeoutError, DEFAULT_OPERATION_TIMEOUTS } from './BaseIntegrationConnector.js';
 import { ConnectorFactory } from './ConnectorFactory.js';
 import { FieldMappingEngine } from './FieldMappingEngine.js';
 import { MatchEngine } from './MatchEngine.js';
@@ -151,22 +153,30 @@ function detectSchemaNotGenerated(entityName, errorMessage) {
         return new SchemaNotGeneratedError(entityName, postgres[1]);
     return null;
 }
+/**
+ * Coerces an externally-supplied tuning value — a duration in ms, or a count — to a usable positive
+ * integer, or `undefined` when it is not one, so the caller's `??` chain falls through to the next
+ * source.
+ *
+ * Every source of these values is outside the engine's control: operator-authored JSON in
+ * `CompanyIntegration.Configuration`, and connector-authored properties like
+ * `BaseIntegrationConnector.FetchChangesTimeoutMs`, whose declared type (`number | null`) happily
+ * admits `0`, negatives and `NaN` — a connector computing one from an unset env var gets `NaN`
+ * without a type error. Handing any of those to `setTimeout` is silently catastrophic rather than
+ * loud: it coerces them to ~1ms, so every wrapped operation rejects immediately and the object
+ * syncs nothing. Guard BOTH sources, not just the JSON one.
+ *
+ * Exported so the guard itself is unit-testable without standing up a sync.
+ */
+export function PositiveInt(v) {
+    return typeof v === 'number' && Number.isFinite(v) && v > 0 ? Math.floor(v) : undefined;
+}
 export class IntegrationEngine extends BaseSingleton {
     constructor() {
         super();
         this.fieldMappingEngine = new FieldMappingEngine();
         this.matchEngine = new MatchEngine();
         this.watermarkService = new WatermarkService();
-        /**
-         * Per-engine async mutex serializing the DB-WRITE section across concurrently-synced streams.
-         * When a layer runs multiple entity maps in parallel (syncConcurrency > 1), they all share ONE
-         * provider connection whose transaction state is singular — so concurrent BeginTransaction /
-         * SavePoint / Commit calls corrupt each other ("Transaction has not begun", "Cannot roll back
-         * SavePoint"). The fetch phase stays parallel (the real throughput win — it's network-bound);
-         * only the per-batch write transaction is serialized through this lock. Keyed per engine
-         * instance, which owns the shared provider.
-         */
-        this._writeChain = Promise.resolve();
         /** Configurable maximum batch size. Connector batches exceeding this are truncated. */
         this.MaxBatchSize = DEFAULT_BATCH_SIZE;
         /** Per-integration request-spacing chain for the rate limiter (keyed by IntegrationID → last scheduled time). */
@@ -216,13 +226,26 @@ export class IntegrationEngine extends BaseSingleton {
         }
         return raw;
     }
+    /**
+     * Per-run execution context (PR 1 item 7). Each run carries its OWN provider,
+     * ownership service and abort plumbing through the async call chain — there is
+     * deliberately NO shared `_provider` instance field, because a last-writer-wins
+     * provider was the reason concurrent runs could write through each other's
+     * connection. AsyncLocalStorage propagates the context to every helper the run
+     * calls without threading a parameter through ~20 private signatures.
+     */
+    static { this.runContext = new AsyncLocalStorage(); }
+    /** The current run's context, when called from inside a sync run. */
+    get currentRunContext() {
+        return IntegrationEngine.runContext.getStore();
+    }
     /** Registers (or clears, with undefined) the post-sync custom-column promotion hook. */
     SetPostSyncSchemaPromotionCallback(callback) {
         this.postSyncSchemaPromotionCallback = callback;
     }
-    /** Returns the active provider — explicit override if set, otherwise the global default. */
+    /** Returns the active provider — the current run's own provider when inside a run, otherwise the global default. */
     get ProviderToUse() {
-        return this._provider ?? Metadata.Provider;
+        return this.currentRunContext?.provider ?? Metadata.Provider;
     }
     /** In-process lock map to prevent concurrent syncs for the same CompanyIntegration */
     static { this.activeSyncs = new Map(); }
@@ -256,35 +279,195 @@ export class IntegrationEngine extends BaseSingleton {
     static GetMaintenanceLock(companyIntegrationID) {
         return IntegrationEngine.maintenanceLocks.get(companyIntegrationID.toLowerCase());
     }
+    /**
+     * Async mutexes serializing the DB-WRITE section, keyed PER PROVIDER (PR 1 item 7 —
+     * per-run write chains). A provider connection's transaction state is singular, so
+     * concurrent BeginTransaction / SavePoint / Commit calls through the SAME provider
+     * corrupt each other ("Transaction has not begun", "Cannot roll back SavePoint") —
+     * whether the writers are parallel streams within one run OR two different runs that
+     * happen to share a provider instance. Keying the mutex on the provider makes the
+     * serialization boundary exactly the hazard boundary: runs on their own providers
+     * write fully in parallel; anything sharing a connection is serialized. The fetch
+     * phase stays parallel (the real throughput win — it's network-bound). WeakMap so a
+     * retired provider's chain entry is collectable.
+     */
+    static { this.writeChains = new WeakMap(); }
     runWriteExclusive(fn) {
+        const provider = this.ProviderToUse;
+        let holder = IntegrationEngine.writeChains.get(provider);
+        if (!holder) {
+            holder = { chain: Promise.resolve() };
+            IntegrationEngine.writeChains.set(provider, holder);
+        }
         // Run fn after the prior write completes (whether it resolved or rejected); keep the chain
         // alive past failures so one errored batch never deadlocks subsequent writers.
-        const run = this._writeChain.then(() => fn(), () => fn());
-        this._writeChain = run.then(() => undefined, () => undefined);
+        const run = holder.chain.then(() => fn(), () => fn());
+        holder.chain = run.then(() => undefined, () => undefined);
         return run;
     }
-    /** Abort controllers for cancelling running syncs */
-    static { this._abortControllers = new Map(); }
-    /** Live sync progress — updated on every batch for ALL syncs regardless of caller */
-    static { this._syncProgress = new Map(); }
-    /** Read current sync progress for a connector. Returns undefined if no sync is running. */
-    static GetSyncProgress(companyIntegrationID) {
-        return IntegrationEngine._syncProgress.get(companyIntegrationID.toLowerCase());
+    /**
+     * Fence check at a batch boundary, BEFORE any write (PR 1 item 3). One SELECT of the run's
+     * ownership columns: if the fence has moved (the stale sweep or another worker reclaimed the
+     * run) throw RunOwnershipLostError so the loop aborts WITHOUT writing this batch; if a
+     * cross-process cancel was requested, trip the run's abort controller so the loop winds down
+     * through the normal cancel path. A caller with no ownership context (e.g. unit tests driving
+     * internals directly) is a no-op.
+     */
+    async assertOwnershipAtBoundary() {
+        const ctx = this.currentRunContext;
+        if (!ctx?.ownership)
+            return;
+        const check = await ctx.ownership.CheckBoundary();
+        if (!check.Owned) {
+            ctx.ownershipLost = true;
+            ctx.abortController.abort();
+            throw new RunOwnershipLostError(ctx.ownership.RunID, 'fence moved at batch boundary — run was reclaimed by another process');
+        }
+        if (check.CancelRequested && !ctx.cancelRequested) {
+            ctx.cancelRequested = true;
+            ctx.abortController.abort();
+        }
     }
-    /** Cancel a running sync for a connector. Returns true if a sync was found and signalled. */
-    static CancelSync(companyIntegrationID) {
-        const key = companyIntegrationID.toLowerCase();
-        const controller = IntegrationEngine._abortControllers.get(key);
-        if (controller) {
-            console.log(`[IntegrationEngine] Cancelling sync for ${companyIntegrationID}`);
-            controller.abort();
-            return true;
+    /**
+     * One-time-per-method deprecation notice. Repeating it on every call would flood the
+     * log of a caller that polls progress on a timer, which is the usual usage.
+     */
+    static { this._deprecationNoticesEmitted = new Set(); }
+    static noteDeprecated(oldName, replacement) {
+        if (IntegrationEngine._deprecationNoticesEmitted.has(oldName)) {
+            return;
         }
+        IntegrationEngine._deprecationNoticesEmitted.add(oldName);
+        console.warn(`IntegrationEngine.${oldName}() is deprecated and no longer functional: sync progress and ` +
+            `cancellation now live on the CompanyIntegrationRun row so they are visible across processes, ` +
+            `and the in-process map this method read no longer exists. Use ${replacement}() instead.`);
+    }
+    /**
+     * @deprecated Superseded by {@link IntegrationEngine.GetSyncProgressAsync}, which reads
+     * the run row and therefore sees runs owned by ANY process.
+     *
+     * Retained with its original signature so a published consumer does not break on a
+     * minor upgrade. It cannot be made to work: the static map it used to read was removed
+     * when progress moved to the database, and a synchronous method cannot query it. It
+     * returns `undefined` — the same value it returned when no run was in progress — and
+     * logs once explaining the replacement.
+     */
+    static GetSyncProgress(_companyIntegrationID) {
+        IntegrationEngine.noteDeprecated('GetSyncProgress', 'GetSyncProgressAsync');
+        return undefined;
+    }
+    /**
+     * @deprecated Superseded by {@link IntegrationEngine.CancelSyncAsync}, which records the
+     * cancel on the run row so the owning process observes it at its next batch boundary.
+     *
+     * Retained with its original signature for published consumers. Returns `false` —
+     * truthfully reporting that it cancelled nothing — rather than pretending to succeed,
+     * so a caller branching on the result is not silently misled.
+     */
+    static CancelSync(_companyIntegrationID) {
+        IntegrationEngine.noteDeprecated('CancelSync', 'CancelSyncAsync');
         return false;
     }
-    /** Get all active sync progress entries */
+    /**
+     * @deprecated No direct replacement. Query `MJ: Company Integration Runs` for rows whose
+     * Status is `In Progress` or `Queued` and read their `ProgressJSON`, which is what
+     * {@link IntegrationEngine.GetSyncProgressAsync} does for a single connector.
+     *
+     * Retained with its original signature for published consumers; returns an empty map.
+     */
     static GetAllSyncProgress() {
-        return new Map(IntegrationEngine._syncProgress);
+        IntegrationEngine.noteDeprecated('GetAllSyncProgress', 'GetSyncProgressAsync');
+        return new Map();
+    }
+    /**
+     * Read current sync progress for a connector FROM THE DATABASE (PR 1 item 4 —
+     * progress lives on the run row, so it is visible from ANY process, not just the
+     * one executing the sync). Returns undefined when no live run exists. A run is
+     * "live" when Status is In Progress/Queued AND its lease has not expired — an
+     * expired lease means the owner died and the snapshot is stale history, not
+     * progress.
+     */
+    static async GetSyncProgressAsync(companyIntegrationID, contextUser, provider) {
+        // Server-side providers are DatabaseProviderBase (which implements IRunViewProvider);
+        // same narrowing the engine uses for the ownership sprocs.
+        const rv = new RunView(provider);
+        const result = await rv.RunView({
+            EntityName: 'MJ: Company Integration Runs',
+            ExtraFilter: `CompanyIntegrationID='${companyIntegrationID.replace(/'/g, "''")}' AND Status IN ('In Progress','Queued')`,
+            OrderBy: 'StartedAt DESC',
+            Fields: ['ProgressJSON', 'LeaseExpiresAt', 'StartedAt'],
+            MaxRows: 1,
+            ResultType: 'simple',
+            BypassCache: true, // live liveness/progress read — must see the current row
+        }, contextUser);
+        const row = result.Success ? result.Results?.[0] : undefined;
+        if (!row)
+            return undefined;
+        if (row.LeaseExpiresAt != null && new Date(row.LeaseExpiresAt).getTime() < Date.now()) {
+            return undefined; // owner's lease lapsed — not live progress
+        }
+        if (!row.ProgressJSON)
+            return undefined;
+        try {
+            const snapshot = JSON.parse(row.ProgressJSON);
+            return { ...snapshot, StartedAt: new Date(snapshot.StartedAt) };
+        }
+        catch {
+            return undefined; // corrupt snapshot — treat as no progress rather than throwing at a poller
+        }
+    }
+    /**
+     * Request cancellation of a running (or queued) sync by stamping
+     * CancelRequestedAt on the live run row (PR 1 item 4 — the DATABASE is the single
+     * source of cancellation truth, so a cancel issued in ANY process reaches the
+     * owner). The owner observes the stamp at its next batch boundary / lease renewal
+     * and stops after the current batch. Returns true when a live run row was stamped.
+     */
+    static async CancelSyncAsync(companyIntegrationID, contextUser, provider) {
+        const p = (provider ?? Metadata.Provider);
+        const d = p.Dialect;
+        const schema = d.QuoteIdentifier(p.MJCoreSchemaName);
+        const table = d.QuoteIdentifier('CompanyIntegrationRun');
+        const ciCol = d.QuoteIdentifier('CompanyIntegrationID');
+        const statusCol = d.QuoteIdentifier('Status');
+        const cancelCol = d.QuoteIdentifier('CancelRequestedAt');
+        const placeholder = p.BuildParameterPlaceholder(0);
+        // One UPDATE, no select-then-update: stamp every un-stamped live run for this CI.
+        const rows = await p.ExecuteSQL(`UPDATE ${schema}.${table} SET ${cancelCol} = ${d.CurrentTimestampUTC()} ` +
+            `WHERE ${ciCol} = ${placeholder} AND ${statusCol} IN ('In Progress','Queued') AND ${cancelCol} IS NULL`, [companyIntegrationID], { isMutation: true, description: 'CancelSync — stamp CancelRequestedAt' }, contextUser);
+        console.log(`[IntegrationEngine] Cancel requested for ${companyIntegrationID} (DB stamp)`);
+        // ExecuteSQL result shape for UPDATE differs per driver; a thrown error is the failure
+        // signal. Verify via a cheap read so the caller gets an honest "was anything live?".
+        void rows;
+        const check = await p.ExecuteSQL(`SELECT COUNT(*) AS N FROM ${schema}.${table} WHERE ${ciCol} = ${placeholder} ` +
+            `AND ${statusCol} IN ('In Progress','Queued') AND ${cancelCol} IS NOT NULL`, [companyIntegrationID], { isMutation: false, description: 'CancelSync — verify stamp' }, contextUser);
+        return Number(check?.[0]?.N ?? 0) > 0;
+    }
+    /**
+     * Worker-mode poll (PR 1 item 8): the oldest claimable `Queued` runs. A row is claimable
+     * when it is unowned or its lease has lapsed — a Queued row with a LIVE lease is being
+     * started by another worker right now and must not be returned. This is only a
+     * *candidate* list; {@link ExecuteQueuedRun}'s atomic claim is what actually grants
+     * exclusivity, so two workers polling simultaneously is safe by construction.
+     */
+    static async PollQueuedRuns(contextUser, maxRows, provider) {
+        const p = (provider ?? Metadata.Provider);
+        const now = p.Dialect.CurrentTimestampUTC();
+        const rv = new RunView(p);
+        const result = await rv.RunView({
+            EntityName: 'MJ: Company Integration Runs',
+            ExtraFilter: `Status='Queued' AND (OwnerToken IS NULL OR LeaseExpiresAt IS NULL OR LeaseExpiresAt < ${now})`,
+            OrderBy: 'StartedAt ASC',
+            Fields: ['ID', 'CompanyIntegrationID'],
+            MaxRows: maxRows,
+            ResultType: 'simple',
+            BypassCache: true, // queue read — must see the current rows
+        }, contextUser);
+        if (!result.Success) {
+            console.warn(`[IntegrationEngine] Queue poll failed: ${result.ErrorMessage}`);
+            return [];
+        }
+        return result.Results ?? [];
     }
     /**
      * U3 — pure, MONOTONIC progress fold: applies one per-map progress event to the live
@@ -306,13 +489,17 @@ export class IntegrationEngine extends BaseSingleton {
      * Call this once during MJAPI startup after metadata is loaded.
      */
     async ResumeOrphanedSyncs(contextUser, provider) {
-        if (provider)
-            this._provider = provider;
+        const prov = provider ?? Metadata.Provider;
         await IntegrationEngineBase.Instance.Config(false, contextUser, provider);
+        // Liveness pre-screen (PR 1 item 6): an 'In Progress' run with a LIVE lease belongs to a
+        // healthy process — possibly another worker — and must not be adopted. Only unowned runs or
+        // runs whose lease has lapsed are orphan CANDIDATES. The claim sproc below re-evaluates the
+        // same predicate atomically, so this filter is a cheap pre-screen, not the correctness gate.
+        const dialect = prov.Dialect;
         const rv = new RunView();
         const orphanedRuns = await rv.RunView({
             EntityName: 'MJ: Company Integration Runs',
-            ExtraFilter: `Status='In Progress'`,
+            ExtraFilter: `Status='In Progress' AND (OwnerToken IS NULL OR LeaseExpiresAt IS NULL OR LeaseExpiresAt < ${dialect.CurrentTimestampUTC()})`,
             ResultType: 'entity_object',
             BypassCache: true, // resume must see the live in-progress runs, not a stale cache
         }, contextUser);
@@ -339,7 +526,18 @@ export class IntegrationEngine extends BaseSingleton {
             let resolveResumeLock;
             let resumeResult;
             IntegrationEngine.activeSyncs.set(lockKey, new Promise(res => { resolveResumeLock = res; }));
+            const ownership = new RunOwnershipService(prov, runID, undefined, contextUser);
             try {
+                // CLAIM BEFORE ADOPTING (PR 1 item 6): a single atomic UPDATE that succeeds only if the
+                // run is still unowned/lapsed. Zero rows = another worker adopted it between our RunView
+                // and now — skip, never double-run. A successful claim BUMPS the fence, so if the
+                // original owner is actually alive-but-slow it aborts at its next boundary check
+                // without writing: the sweep-reclaim is itself the abort signal for the abandoned owner.
+                const claimed = await ownership.Claim();
+                if (!claimed) {
+                    console.log(`[IntegrationEngine] Skipping resume of run ${runID.substring(0, 8)} — claim lost (another worker adopted it)`);
+                    continue;
+                }
                 // Find which entity MAPS already completed SUCCESSFULLY in this run. We correlate
                 // by EntityMapID (parsed from the detail's RecordID, stamped by CreateRunDetail),
                 // not EntityID — two maps can target the same MJ Entity, so keying on EntityID
@@ -367,23 +565,78 @@ export class IntegrationEngine extends BaseSingleton {
                 console.log(`[IntegrationEngine] Resuming run ${runID.substring(0, 8)}... ` +
                     `for ${companyIntegrationID.substring(0, 8)}... ` +
                     `(${completedMapIDs.size} entity maps already completed)`);
+                // Recover what this run was ASKED to do. Without this the resume rebuilds config from
+                // the CompanyIntegration alone, so an adopted run silently loses its options — most
+                // damagingly FullSync, which exists precisely to distrust the watermark. An adopted
+                // full sync would resume incrementally, fetch nothing, and report Success.
+                // Unparseable/absent ConfigData falls back to defaults rather than refusing to resume.
+                let resumeOptions;
+                let resumeTriggerType = 'Scheduled';
+                try {
+                    const cfg = JSON.parse(run.ConfigData ?? '{}');
+                    resumeOptions = cfg.options ?? undefined;
+                    if (cfg.triggerType)
+                        resumeTriggerType = cfg.triggerType;
+                }
+                catch {
+                    console.warn(`[IntegrationEngine] Run ${runID.substring(0, 8)} has unparseable ConfigData; resuming with defaults`);
+                }
+                if (resumeOptions?.FullSync) {
+                    console.log(`[IntegrationEngine] Run ${runID.substring(0, 8)} was a FULL sync — resuming as full, not incremental`);
+                }
                 // Load config and filter to only remaining entity maps (by map ID)
-                const config = await this.LoadRunConfiguration(companyIntegrationID, contextUser);
+                const config = await this.LoadRunConfiguration(companyIntegrationID, contextUser, resumeOptions);
                 const remainingMaps = config.entityMaps.filter(em => !completedMapIDs.has(em.ID.toLowerCase()));
                 if (remainingMaps.length === 0) {
                     console.log(`[IntegrationEngine] All entity maps completed for run ${runID.substring(0, 8)}, marking as Success`);
                     run.EndedAt = new Date();
                     run.Status = 'Success';
+                    ownership.SyncEntityOwnershipFields(run); // full-row save must not clobber the live claim
                     await run.Save();
+                    await ownership.Release('Success');
                     continue;
                 }
                 console.log(`[IntegrationEngine] Resuming ${remainingMaps.length} remaining entity maps (of ${config.entityMaps.length} total)`);
                 // Replace entityMaps with only the remaining ones
                 config.entityMaps = remainingMaps;
-                // Execute remaining maps using the existing run record
-                const result = await this.ExecuteEntityMaps(config, run, contextUser);
-                result.RunID = runID;
-                await this.FinalizeRun(run, result, contextUser);
+                // Execute remaining maps inside a per-run context: the resume gets its own provider
+                // binding, abort controller, and ownership — identical to a fresh RunSync — so the
+                // heartbeat renews the lease, the batch boundaries fence-check, and FinalizeRun
+                // syncs ownership fields + releases, all through the SAME code paths.
+                const abortController = new AbortController();
+                const progressSnapshot = {
+                    StartedAt: new Date(),
+                    CurrentEntity: '',
+                    EntityMapsTotal: remainingMaps.length,
+                    EntityMapsCompleted: 0,
+                    RecordsProcessed: 0,
+                    RecordsCreated: 0,
+                    RecordsUpdated: 0,
+                    RecordsErrored: 0,
+                    // The run's OWN trigger type, recovered above — not a hardcoded 'Scheduled'. This is
+                    // what IntegrationGetSyncProgress reports back ("Sync in progress (Manual)"), so a
+                    // hardcoded value mislabels every adopted run.
+                    TriggerType: resumeTriggerType,
+                };
+                const runCtx = {
+                    provider: prov,
+                    ownership,
+                    abortController,
+                    progressSnapshot,
+                    cancelRequested: false,
+                    ownershipLost: false,
+                };
+                ownership.StartHeartbeat({
+                    onLost: () => { runCtx.ownershipLost = true; abortController.abort(); },
+                    onCancelRequested: () => { runCtx.cancelRequested = true; abortController.abort(); },
+                    progressSupplier: () => JSON.stringify(progressSnapshot),
+                });
+                const result = await IntegrationEngine.runContext.run(runCtx, async () => {
+                    const r = await this.ExecuteEntityMaps(config, run, contextUser, undefined, abortController.signal);
+                    r.RunID = runID;
+                    await this.FinalizeRun(run, r, contextUser);
+                    return r;
+                });
                 resumeResult = result;
                 console.log(`[IntegrationEngine] Resume complete for ${runID.substring(0, 8)}: ` +
                     `${result.RecordsCreated} created, ${result.RecordsUpdated} updated, ` +
@@ -392,13 +645,26 @@ export class IntegrationEngine extends BaseSingleton {
             catch (err) {
                 const errMsg = err instanceof Error ? err.message : String(err);
                 console.error(`[IntegrationEngine] Failed to resume run ${runID.substring(0, 8)}: ${errMsg}`);
-                // Mark as failed so it doesn't get picked up again
-                run.EndedAt = new Date();
-                run.Status = 'Failed';
-                run.ErrorLog = JSON.stringify([{ ErrorMessage: `Resume failed: ${errMsg}` }]);
-                await run.Save();
+                if (err instanceof RunOwnershipLostError) {
+                    // We were fenced out mid-resume — the NEW owner now owns the run row.
+                    // Writing 'Failed' here would clobber the live holder's state.
+                    console.warn(`[IntegrationEngine] Resume of run ${runID.substring(0, 8)} lost ownership — leaving the run row to its new owner`);
+                }
+                else {
+                    // Mark as failed so it doesn't get picked up again
+                    run.EndedAt = new Date();
+                    run.Status = 'Failed';
+                    run.ErrorLog = JSON.stringify([{ ErrorMessage: `Resume failed: ${errMsg}` }]);
+                    ownership.SyncEntityOwnershipFields(run);
+                    await run.Save();
+                    try {
+                        await ownership.Release('Failed');
+                    }
+                    catch { /* lease will simply expire */ }
+                }
             }
             finally {
+                ownership.StopHeartbeat();
                 // Release the C1 lock + unblock any RunSync that began awaiting this resume (RunSync returns
                 // `existing`). Resolve with the real result when we have one, else a benign empty result so no
                 // waiter hangs. Promise resolve is idempotent and the early-exit `continue` also lands here.
@@ -424,8 +690,98 @@ export class IntegrationEngine extends BaseSingleton {
      * @returns Aggregate sync result with record counts and errors
      */
     async RunSync(companyIntegrationID, contextUser, triggerType = 'Manual', onProgress, onNotification, options, provider) {
-        if (provider)
-            this._provider = provider;
+        return this.runWithOwnedContext(companyIntegrationID, contextUser, triggerType, onProgress, onNotification, options, provider);
+    }
+    /**
+     * Worker-mode enqueue (PR 1 item 8): create the run row with `Status='Queued'` and
+     * return its ID immediately, without executing anything. A worker process picks it up
+     * via {@link ExecuteQueuedRun}; the claim sproc provides the mutual exclusion, so any
+     * number of workers can poll the same queue safely.
+     *
+     * The trigger type and sync options are persisted on the run's `ConfigData` so the
+     * worker executes exactly what the caller asked for, in a different process.
+     */
+    async EnqueueSync(companyIntegrationID, contextUser, triggerType = 'Manual', options, provider) {
+        const md = provider ?? Metadata.Provider;
+        const run = await md.GetEntityObject('MJ: Company Integration Runs', contextUser);
+        run.NewRecord();
+        run.CompanyIntegrationID = companyIntegrationID;
+        run.RunByUserID = contextUser.ID;
+        run.StartedAt = new Date();
+        run.Status = 'Queued';
+        run.TotalRecords = 0;
+        run.ConfigData = JSON.stringify({ triggerType, options: options ?? null });
+        if (options?.ScheduledJobRunID) {
+            run.Set('ScheduledJobRunID', options.ScheduledJobRunID);
+        }
+        if (!(await run.Save())) {
+            throw new Error(`Failed to enqueue sync run: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+        }
+        console.log(`[IntegrationEngine] Enqueued run ${run.ID} for ${companyIntegrationID} (${triggerType})`);
+        return run.ID;
+    }
+    /**
+     * Worker-mode execution (PR 1 item 8): take a `Queued` run row, claim it, and execute
+     * it in THIS process. Returns a failed result (without side effects) when the run is
+     * no longer queued or when another worker won the claim — losing the race is normal
+     * and must never be treated as an error condition by the caller's poll loop.
+     */
+    async ExecuteQueuedRun(runID, contextUser, provider) {
+        const md = provider ?? Metadata.Provider;
+        const run = await md.GetEntityObject('MJ: Company Integration Runs', contextUser);
+        if (!(await run.Load(runID))) {
+            return this.emptyFailedResult(`Queued run ${runID} not found`);
+        }
+        if (run.Status !== 'Queued') {
+            return this.emptyFailedResult(`Run ${runID} is '${run.Status}', not 'Queued' — another worker already took it`);
+        }
+        // A cancel issued while the run was still QUEUED must stop it here. CancelSync stamps
+        // both 'In Progress' and 'Queued' rows, but the only consumer of the stamp is the
+        // running loop's batch-boundary / lease-renewal check — a queued run has no loop yet,
+        // so without this gate the worker claims the cancelled row moments later and executes
+        // it to completion. Verified live: run E3F51F9A was stamped CancelRequestedAt at
+        // 15:50:46.643 and still finished Status='Success' at 15:50:48.646.
+        // Finalize the same way an aborted in-flight run finalizes (FinalizeRun): 'Cancelled'
+        // with an explicit ErrorLog carrying the reason.
+        if (run.CancelRequestedAt != null) {
+            run.EndedAt = new Date();
+            run.Status = 'Cancelled';
+            run.ErrorLog = 'Sync cancelled by user before it started';
+            if (!(await run.Save())) {
+                console.warn(`[IntegrationEngine] Could not finalize cancelled queued run ${runID}: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+            }
+            console.log(`[IntegrationEngine] Queued run ${runID} was cancelled before start — not executing`);
+            return this.emptyFailedResult('Sync cancelled by user before it started');
+        }
+        let triggerType = 'Scheduled';
+        let options;
+        try {
+            const config = JSON.parse(run.ConfigData ?? '{}');
+            if (config.triggerType)
+                triggerType = config.triggerType;
+            options = config.options ?? undefined;
+        }
+        catch {
+            // A run whose ConfigData we can't read still executes — with defaults, not silently skipped.
+            console.warn(`[IntegrationEngine] Run ${runID} has unparseable ConfigData; executing with defaults`);
+        }
+        return this.runWithOwnedContext(run.CompanyIntegrationID, contextUser, triggerType, undefined, undefined, options, provider, run);
+    }
+    /** A zero-work failed SyncResult — used for refusals that must not look like partial work. */
+    emptyFailedResult(message) {
+        return {
+            Success: false, ErrorMessage: message, RecordsProcessed: 0, RecordsCreated: 0,
+            RecordsUpdated: 0, RecordsDeleted: 0, RecordsErrored: 0, RecordsSkipped: 0,
+            Errors: [], EntityMapResults: [], Duration: 0,
+        };
+    }
+    /**
+     * Shared body of {@link RunSync} and {@link ExecuteQueuedRun}: acquires the per-connection
+     * concurrency lock, establishes the per-run AsyncLocalStorage context (own provider, own
+     * abort controller, own progress snapshot) and executes. `existingRun` is supplied by the
+     * worker path so a queued row is executed rather than a new row created.
+     */
+    async runWithOwnedContext(companyIntegrationID, contextUser, triggerType, onProgress, onNotification, options, provider, existingRun) {
         const lockKey = companyIntegrationID.toLowerCase();
         const existing = IntegrationEngine.activeSyncs.get(lockKey);
         if (existing) {
@@ -446,10 +802,12 @@ export class IntegrationEngine extends BaseSingleton {
                 Errors: [], EntityMapResults: [], Duration: 0,
             };
         }
-        // Initialize abort controller and progress tracking
+        // Per-run context (PR 1 item 7): the run's OWN provider (captured, never stored on
+        // the engine), a local abort controller (plumbing driven by the DB cancel/fence
+        // signals — never a cross-process source of truth), and a local progress snapshot
+        // that the ownership service persists to ProgressJSON on the run row.
         const abortController = new AbortController();
-        IntegrationEngine._abortControllers.set(lockKey, abortController);
-        IntegrationEngine._syncProgress.set(lockKey, {
+        const progressSnapshot = {
             StartedAt: new Date(),
             CurrentEntity: '',
             EntityMapsTotal: 0,
@@ -459,33 +817,41 @@ export class IntegrationEngine extends BaseSingleton {
             RecordsUpdated: 0,
             RecordsErrored: 0,
             TriggerType: triggerType,
-        });
+        };
+        const runCtx = {
+            provider: provider ?? Metadata.Provider,
+            abortController,
+            progressSnapshot,
+            cancelRequested: false,
+            ownershipLost: false,
+        };
         // Wrap caller's onProgress with internal tracking. U3 — MONOTONIC: with
         // syncConcurrency > 1 the per-map events arrive out of order (map 3 can emit after
         // map 7), so raw assignment made the progress bar go BACKWARDS. The snapshot is a
-        // high-water mark, so only ever ratchet the counters upward.
+        // high-water mark, so only ever ratchet the counters upward. The snapshot is
+        // persisted to the run row's ProgressJSON (throttled) so readers in ANY process see it.
         const wrappedProgress = (progress) => {
-            const entry = IntegrationEngine._syncProgress.get(lockKey);
-            if (entry)
-                IntegrationEngine.RatchetProgressSnapshot(entry, progress);
+            IntegrationEngine.RatchetProgressSnapshot(progressSnapshot, progress);
+            void runCtx.ownership?.WriteProgress(JSON.stringify(progressSnapshot));
             if (onProgress)
                 onProgress(progress);
         };
-        const syncPromise = this.executeSyncInternal(companyIntegrationID, contextUser, triggerType, wrappedProgress, onNotification, options, abortController.signal);
+        // Enter the AsyncLocalStorage scope — every helper the run calls resolves
+        // ProviderToUse / write chain / ownership from THIS context, isolated per run.
+        const syncPromise = IntegrationEngine.runContext.run(runCtx, () => this.executeSyncInternal(companyIntegrationID, contextUser, triggerType, wrappedProgress, onNotification, options, abortController.signal, existingRun));
         IntegrationEngine.activeSyncs.set(lockKey, syncPromise);
         try {
             return await syncPromise;
         }
         finally {
             IntegrationEngine.activeSyncs.delete(lockKey);
-            IntegrationEngine._abortControllers.delete(lockKey);
-            IntegrationEngine._syncProgress.delete(lockKey);
+            runCtx.ownership?.StopHeartbeat();
         }
     }
     /**
      * Internal sync execution method. Contains the full orchestration logic.
      */
-    async executeSyncInternal(companyIntegrationID, contextUser, triggerType, onProgress, onNotification, options, abortSignal) {
+    async executeSyncInternal(companyIntegrationID, contextUser, triggerType, onProgress, onNotification, options, abortSignal, existingRun) {
         const startTime = Date.now();
         const logger = new SyncLogger({ ciId: companyIntegrationID, integration: null });
         logger.emit('sync.run.start', {
@@ -523,6 +889,15 @@ export class IntegrationEngine extends BaseSingleton {
         if (config.companyIntegration.IsActive === false) {
             const message = 'Connector is deactivated (IsActive=false); sync not started';
             logger.emit('sync.warning', { reason: 'deactivated', message });
+            // A queued row must not sit in the queue forever being re-polled by every worker.
+            if (existingRun) {
+                existingRun.Status = 'Failed';
+                existingRun.ErrorLog = message;
+                existingRun.EndedAt = new Date();
+                if (!(await existingRun.Save())) {
+                    console.warn(`[IntegrationEngine] Failed to fail-out deactivated queued run ${existingRun.ID}: ${existingRun.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+                }
+            }
             return {
                 Success: false,
                 ErrorMessage: message,
@@ -537,8 +912,53 @@ export class IntegrationEngine extends BaseSingleton {
                 Duration: Date.now() - startTime,
             };
         }
-        const run = await this.CreateRunRecord(config.companyIntegration, triggerType, contextUser, options?.ScheduledJobRunID);
+        // Worker mode executes a row that already exists (Status='Queued'); the direct path creates one.
+        const run = existingRun ?? await this.CreateRunRecord(config.companyIntegration, triggerType, contextUser, options?.ScheduledJobRunID, options);
         logger.attachRunId(run.ID);
+        // ── Durable-run ownership (PR 1 item 3): claim before the first batch. ──
+        // The claim is ONE atomic UPDATE (unowned OR expired lease) that bumps FenceToken;
+        // zero rows back means another process owns this run and we must not proceed.
+        // Renewal runs from a TIMER at ~lease/3 (not the batch loop) so a long batch never
+        // looks dead; the lease is max(default, MaxRuntimeMinutes) so a configured long run
+        // only ever EXTENDS protection. The heartbeat's renewal result doubles as the
+        // cross-process cancel poll.
+        const runCtx = this.currentRunContext;
+        if (runCtx) {
+            const ownership = new RunOwnershipService(runCtx.provider, run.ID, options?.MaxRuntimeMinutes ?? undefined, contextUser);
+            const claimed = await ownership.Claim();
+            if (!claimed) {
+                const message = `Run ${run.ID} could not be claimed — another process holds a live lease. Not proceeding.`;
+                logger.emit('sync.warning', { reason: 'claim-lost', message });
+                return {
+                    Success: false, ErrorMessage: message, RecordsProcessed: 0, RecordsCreated: 0,
+                    RecordsUpdated: 0, RecordsDeleted: 0, RecordsErrored: 0, RecordsSkipped: 0,
+                    Errors: [], EntityMapResults: [], Duration: Date.now() - startTime, RunID: run.ID,
+                };
+            }
+            runCtx.ownership = ownership;
+            // Worker mode: the claim is what promotes a Queued row to In Progress. Doing it
+            // AFTER the claim (never before) means a worker that loses the race never touches
+            // the row, so the winner's status is the only one written.
+            if (existingRun && existingRun.Status === 'Queued') {
+                existingRun.Status = 'In Progress';
+                existingRun.StartedAt = new Date();
+                ownership.SyncEntityOwnershipFields(existingRun);
+                if (!(await existingRun.Save())) {
+                    console.warn(`[IntegrationEngine] Failed to mark claimed run ${existingRun.ID} In Progress: ${existingRun.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+                }
+            }
+            ownership.StartHeartbeat({
+                onLost: () => {
+                    runCtx.ownershipLost = true;
+                    runCtx.abortController.abort();
+                },
+                onCancelRequested: () => {
+                    runCtx.cancelRequested = true;
+                    runCtx.abortController.abort();
+                },
+                progressSupplier: () => JSON.stringify(runCtx.progressSnapshot),
+            });
+        }
         // Durable, queryable, restart-surviving artifact stream for this sync. runID is
         // the CompanyIntegrationRun.ID so the JSONL artifact cross-correlates with the run
         // row. Exposed over GraphQL (IntegrationListRuns / IntegrationGetRun /
@@ -603,6 +1023,20 @@ export class IntegrationEngine extends BaseSingleton {
         catch (err) {
             const errMsg = err instanceof Error ? err.message : String(err);
             logger.emit('sync.run.fail', { error: errMsg, durationMs: Date.now() - startTime });
+            // Ownership lost (fence moved / lease reclaimed): the run row now belongs to
+            // ANOTHER process — writing a terminal status to it here would clobber the new
+            // owner's state. Stop everything locally and walk away without touching the row.
+            const ctx = this.currentRunContext;
+            if (err instanceof RunOwnershipLostError || ctx?.ownershipLost) {
+                ctx?.ownership?.StopHeartbeat();
+                await this.finalizeSyncProgress(progress, 'failed', errMsg);
+                console.warn(`[IntegrationEngine] Run ${run.ID} ownership lost — aborted without writing the run row.`);
+                return {
+                    Success: false, ErrorMessage: errMsg, RecordsProcessed: 0, RecordsCreated: 0,
+                    RecordsUpdated: 0, RecordsDeleted: 0, RecordsErrored: 0, RecordsSkipped: 0,
+                    Errors: [], EntityMapResults: [], Duration: Date.now() - startTime, RunID: run.ID,
+                };
+            }
             await this.finalizeSyncProgress(progress, 'failed', errMsg);
             await this.FailRun(run, err, contextUser, onNotification);
             throw err;
@@ -771,7 +1205,7 @@ export class IntegrationEngine extends BaseSingleton {
     /**
      * Creates a new CompanyIntegrationRun record to track this sync.
      */
-    async CreateRunRecord(companyIntegration, triggerType, contextUser, scheduledJobRunID) {
+    async CreateRunRecord(companyIntegration, triggerType, contextUser, scheduledJobRunID, options) {
         const md = this.ProviderToUse;
         const run = await md.GetEntityObject('MJ: Company Integration Runs', contextUser);
         run.NewRecord();
@@ -780,7 +1214,12 @@ export class IntegrationEngine extends BaseSingleton {
         run.StartedAt = new Date();
         run.Status = 'In Progress';
         run.TotalRecords = 0;
-        run.ConfigData = JSON.stringify({ triggerType });
+        // Persist the OPTIONS alongside the trigger type, in the same shape EnqueueSync writes.
+        // Without them a run that outlives its process loses what it was asked to do: ResumeOrphanedSyncs
+        // rebuilds config from the CompanyIntegration alone, so an adopted `FullSync` run silently
+        // resumed as an incremental one — re-fetching nothing and reporting Success, which is the
+        // opposite of what a full sync is requested for (repairing drift, re-pulling after a remap).
+        run.ConfigData = JSON.stringify({ triggerType, options: options ?? null });
         // Link to scheduled job run if triggered by the scheduler.
         // Use Set() because the ScheduledJobRunID column won't exist on the
         // generated entity type until CodeGen runs after the migration.
@@ -848,6 +1287,11 @@ export class IntegrationEngine extends BaseSingleton {
                 return { ok: mapResult.Success, throttled: mapResult.Throttled === true };
             }
             catch (err) {
+                // Ownership loss is NOT a per-map failure to record-and-continue: continuing to the
+                // next map would keep writing after another process claimed the run — the exact
+                // split-brain the fence prevents. Propagate so the whole sync aborts immediately.
+                if (err instanceof RunOwnershipLostError)
+                    throw err;
                 const objName = entityMap.ExternalObjectName ?? entityMap.ID;
                 const errMsg = err instanceof Error ? err.message : String(err);
                 console.error(`[IntegrationEngine] Entity map '${objName}' failed: ${errMsg}`);
@@ -1282,12 +1726,12 @@ export class IntegrationEngine extends BaseSingleton {
             if (!raw)
                 return {};
             const p = JSON.parse(raw);
-            const num = (v) => (typeof v === 'number' && Number.isFinite(v) && v > 0 ? Math.floor(v) : undefined);
             return {
-                maxConcurrency: num(p.maxConcurrency),
+                maxConcurrency: PositiveInt(p.maxConcurrency),
                 rateLimitTokensPerSec: typeof p.rateLimitTokensPerSec === 'number' && p.rateLimitTokensPerSec > 0 ? p.rateLimitTokensPerSec : undefined,
-                rateLimitBurst: num(p.rateLimitBurst),
-                discoveryTimeBudgetMs: num(p.discoveryTimeBudgetMs),
+                rateLimitBurst: PositiveInt(p.rateLimitBurst),
+                discoveryTimeBudgetMs: PositiveInt(p.discoveryTimeBudgetMs),
+                fetchTimeoutMs: PositiveInt(p.fetchTimeoutMs),
             };
         }
         catch {
@@ -1476,6 +1920,7 @@ export class IntegrationEngine extends BaseSingleton {
         let fetchGapCount = 0; // CONSECUTIVE skipped pages (reset on any clean fetch)
         const MAX_FETCH_GAPS = 25; // give up + hold the watermark if this many pages fail in a row (API down)
         let consecutiveEmptyBatches = 0; // P3-D: detect a connector that pages empty-but-HasMore forever
+        let oversizeBatchWarned = false; // pagination rule: warn ONCE per object that the connector ignored BatchSize
         const MAX_BATCHES_PER_MAP = 5000;
         const EMPTY_BATCH_WARN_THRESHOLD = 5; // warn once after this many empty-but-HasMore batches in a row
         const fetchedExternalIDs = new Set(); // Track all IDs seen during this pull for orphan detection
@@ -1486,6 +1931,17 @@ export class IntegrationEngine extends BaseSingleton {
         // one entry per distinct key + a capped value sample. See CustomKeyStat.
         const customKeyAgg = new Map();
         let customKeyTotalRecords = 0;
+        // Per-page fetch timeout, resolved ONCE per entity map. A connector that fans out one request
+        // per parent does N requests inside a single FetchChanges call, so its page time scales with
+        // BatchSize and with however much concurrency the adaptive controller currently allows — the
+        // fixed 30s default punished exactly those connectors. Deployment config wins over the
+        // connector's own declared default, which wins over the framework default. BOTH overrides go
+        // through PositiveInt: `??` alone would only reject null/undefined, so a connector returning
+        // 0 / -1 / NaN (all legal for its `number | null` type) would be applied verbatim and time
+        // every page out at ~1ms.
+        const fetchTimeoutMs = PositiveInt(this.getConfigOverrides(config).fetchTimeoutMs)
+            ?? PositiveInt(config.connector.FetchChangesTimeoutMs)
+            ?? DEFAULT_OPERATION_TIMEOUTS.FetchChangesMs;
         while (hasMore) {
             if (abortSignal?.aborted) {
                 console.log(`[IntegrationEngine] Sync cancelled for ${entityMap.ExternalObjectName} after ${recordsInMap} records — saving watermark`);
@@ -1533,7 +1989,21 @@ export class IntegrationEngine extends BaseSingleton {
                 // Resilient fetch: bound each attempt with a timeout (a hung vendor API must not
                 // hold the sync lock forever) and retry only transient errors (network/throttle/DB).
                 // A non-retryable error (auth, 4xx, parse) throws immediately as before.
-                batch = await WithRetry(() => WithTimeout(config.connector.FetchChanges(ctx), DEFAULT_OPERATION_TIMEOUTS.FetchChangesMs, `FetchChanges(${entityMap.ExternalObjectName})`), undefined, (err) => IsRetryableError(ClassifyError(err).Code), (attempt, err, delayMs) => logger?.emit('sync.fetch.retry', {
+                batch = await WithRetry(() => WithTimeout(config.connector.FetchChanges(ctx), fetchTimeoutMs, `FetchChanges(${entityMap.ExternalObjectName})`), undefined, 
+                // OUR OWN timeout is terminal for this page; a transport error is not.
+                //
+                // `WithTimeout` is a `Promise.race` with no cancellation, so the abandoned attempt
+                // keeps running. Retrying meant a second full page of vendor requests overlapping
+                // the first, then a third — up to 3x the load on a source that was already too slow
+                // to finish once, which is a good way to earn a real 429 (and THAT does cut
+                // concurrency). And the retry could not succeed on its merits anyway: the same work
+                // under the same budget exceeds it again.
+                //
+                // Deliberately `instanceof` rather than the classified code. `ClassifyError` folds
+                // `econnreset` in with timeouts under `NETWORK_TIMEOUT`, and a reset socket IS worth
+                // retrying — so excluding the whole code would lose real resilience. Only the error
+                // WithTimeout itself minted is excluded.
+                (err) => !(err instanceof OperationTimeoutError) && IsRetryableError(ClassifyError(err).Code), (attempt, err, delayMs) => logger?.emit('sync.fetch.retry', {
                     externalObjectName: entityMap.ExternalObjectName,
                     batchIndex: batchCount,
                     attempt,
@@ -1583,7 +2053,33 @@ export class IntegrationEngine extends BaseSingleton {
                         currentPage += 1;
                     continue;
                 }
+                // Cannot skip past this page (cursor paging, or the gap budget is spent), so the object
+                // stops here with an incomplete result set. The watermark is held below and the run
+                // re-fetches next time — but WITHOUT this warning that outcome is invisible: the map
+                // reports success with the records it did get, so an object whose very first page
+                // failed reads as a clean "0 records, nothing changed" run. Verified live: a sync
+                // whose only page timed out finished Status=Success, errorCount=0, empty ErrorLog.
                 fetchCompletedCleanly = false;
+                const abortMessage = `Fetch for '${entityMap.ExternalObjectName}' stopped at batch ${batchCount} after a persistent ` +
+                    `error and could not continue past it, so this object's result set is INCOMPLETE ` +
+                    `(${recordsInMap} record(s) fetched before the failure). The watermark is held, so the ` +
+                    `unfetched window is retried next run. Error: ${errMsg}`;
+                logger?.warning(entityMap.ExternalObjectName ?? entityMap.ID, 'FETCH_ABORTED_INCOMPLETE', abortMessage, { batchIndex: batchCount, recordsFetchedBeforeFailure: recordsInMap, error: errMsg });
+                // The structured warning above reaches the console and the per-run artifact — neither of
+                // which is queryable run history. The DURABLE record is CompanyIntegrationRun, whose
+                // Status is derived from RecordsErrored (unchanged here, correctly: no record failed) and
+                // whose ErrorLog is written from result.Errors. Without an entry there, a nightly sync
+                // that aborts on its first page every night reads as an unbroken run of clean Successes
+                // with TotalRecords=0. Severity 'Warning' is what keeps Status='Success': MergeResult
+                // only clears Success on RecordsErrored > 0, so this records the condition without
+                // reclassifying a held-watermark retry as a failed run.
+                result.Errors.push({
+                    ExternalID: '',
+                    ChangeType: 'Skip',
+                    ErrorMessage: abortMessage,
+                    ErrorCode: 'CONNECTOR_ERROR',
+                    Severity: 'Warning',
+                });
                 break;
             }
             logger?.emit('sync.fetch.batch.complete', {
@@ -1605,11 +2101,15 @@ export class IntegrationEngine extends BaseSingleton {
                     logger?.warning(entityMap.ExternalObjectName ?? 'sync', w.Code, w.Message, w.Data);
                 }
             }
-            // If the connector returned more records than MaxBatchSize, log it but never truncate —
-            // all records are written, just in sub-batches to keep DB transactions manageable.
-            if (batch.Records.length > this.MaxBatchSize) {
-                console.log(`[IntegrationEngine] ${entityMap.ExternalObjectName}: connector returned ` +
-                    `${batch.Records.length} records (> MaxBatchSize ${this.MaxBatchSize}), writing in chunks.`);
+            // Engine-side half of the pagination rule: a connector MUST honour ctx.BatchSize. We never
+            // truncate — every record is written, just in sub-batches to keep DB transactions manageable —
+            // but an over-size batch is a real connector defect and has to be visible on the structured
+            // run-event stream, not buried in a console.log nobody reads. Warned ONCE per object (the
+            // CONSECUTIVE_EMPTY_BATCHES pattern) so a paginating-but-over-size connector doesn't flood
+            // the artifact with one warning per page.
+            if (batch.Records.length > this.MaxBatchSize && !oversizeBatchWarned) {
+                oversizeBatchWarned = true;
+                this.warnOversizedBatch(entityMap, batch, batchCount, logger);
             }
             if (batch.Records.length > 0) {
                 const fingerprint = batch.Records.map(r => r.ExternalID).join(',');
@@ -1653,6 +2153,9 @@ export class IntegrationEngine extends BaseSingleton {
                         `for this object to stream via per-record content-hash instead.`);
                 }
             }
+            // Batch boundary: verify we STILL own the run before this batch's writes begin.
+            // Throws RunOwnershipLostError (aborting with nothing written) if the fence moved.
+            await this.assertOwnershipAtBoundary();
             // Serialize the match READ too (record-map / PK lookups). On a shared provider connection
             // a read routes through whatever transaction is active, so a match read in this stream
             // collides with another concurrent stream's in-flight write transaction ("Transaction has
@@ -1819,6 +2322,26 @@ export class IntegrationEngine extends BaseSingleton {
             await this.runWriteExclusive(() => this.watermarkService.SaveKeysetPosition(entityMapID, currentAfterKey, contextUser));
             result.WatermarkAfter = currentAfterKey;
         }
+        else if (!hadFetchGap && currentWatermark && currentWatermark !== initialWatermark) {
+            // A WATERMARK-based connector stopped early (cancel / safety limit / duplicate batch /
+            // schema-not-generated / unskippable fetch error) but whole batches DID complete. Persist the
+            // max watermark seen so the next run resumes from there instead of re-fetching everything
+            // since the last clean run — the counterpart of the keyset branch above, which until now was
+            // the ONLY early-stop that saved its position. The cancel log has always said "saving
+            // watermark"; for a non-keyset connector it previously saved nothing.
+            //
+            // Safe because currentWatermark only ever advances at the END of a fully-applied batch
+            // (§10, after ApplyRecords) — every early-exit `break` above happens before that, so this
+            // value can never point past a record that wasn't written.
+            //
+            // Deliberately NOT wall-clock "now", even for a full sync: coverage is partial, so advancing
+            // past the point actually reached would skip the (reached, now] window permanently. And
+            // deliberately NOT when hadFetchGap — a skipped page leaves a HOLE behind this watermark,
+            // which is why that path holds it for a full re-fetch next run.
+            const partialWatermark = currentWatermark;
+            await this.runWriteExclusive(() => this.watermarkService.Update(entityMapID, partialWatermark, contextUser, 'Pull'));
+            result.WatermarkAfter = partialWatermark;
+        }
         // Orphan detection: delete/tombstone MJ records whose external counterpart no longer exists.
         // Runs on a full sync OR a partition-reconcile (both fetch the COMPLETE set, so an MJ record
         // whose ExternalID isn't in fetchedExternalIDs is genuinely gone — even one inside an otherwise
@@ -1857,6 +2380,59 @@ export class IntegrationEngine extends BaseSingleton {
         await this.CreateRunDetail(run, entityMap, result, contextUser);
         return result;
     }
+    /**
+     * Classifies a fetched batch against the `ctx.BatchSize` the engine asked for. Returns null when
+     * the batch is within contract, so the caller can treat "no verdict" as "nothing to warn about".
+     *
+     * Two severities, because they are different defects:
+     *  - `CONNECTOR_UNBOUNDED_BATCH` — the FIRST batch came back over-size AND `HasMore` is not true,
+     *    i.e. the connector ignored pagination entirely and pulled the whole object into memory in one
+     *    request. That is what OOMs a large tenant, and it grows silently with the customer's data.
+     *  - `CONNECTOR_IGNORED_BATCH_SIZE` — the connector IS paging but overshoots the requested size
+     *    (e.g. a hardcoded page size). Bounded memory, still a contract violation worth fixing.
+     *
+     * Public + static because the decision is the interesting part and deserves a unit test that
+     * doesn't have to stand up a whole sync (same rationale as {@link RatchetProgressSnapshot}).
+     */
+    static ClassifyOversizedBatch(objectName, recordCount, requestedBatchSize, batchIndex, hasMore) {
+        if (recordCount <= requestedBatchSize)
+            return null;
+        const unbounded = batchIndex === 1 && hasMore !== true;
+        return unbounded
+            ? {
+                Code: 'CONNECTOR_UNBOUNDED_BATCH',
+                Unbounded: true,
+                Message: `'${objectName}': connector returned ALL ${recordCount} records in a single batch ` +
+                    `(requested ${requestedBatchSize}, HasMore=false) — pagination is not implemented for this ` +
+                    `object, so the entire object is held in memory. Every record was written, in chunks.`,
+            }
+            : {
+                Code: 'CONNECTOR_IGNORED_BATCH_SIZE',
+                Unbounded: false,
+                Message: `'${objectName}': connector returned ${recordCount} records for batch ${batchIndex} ` +
+                    `(requested ${requestedBatchSize}) — ctx.BatchSize is not being honoured. Every record was ` +
+                    `written, in chunks. Warned once per object.`,
+            };
+    }
+    /**
+     * Surfaces an over-size batch onto the structured run-event stream (queryable via
+     * IntegrationTailRunEvents) so the pagination-rule violation is visible instead of console-only.
+     * Never truncates and never fails the sync — the over-size batch is written in chunks either way.
+     */
+    warnOversizedBatch(entityMap, batch, batchIndex, logger) {
+        const objectName = entityMap.ExternalObjectName ?? entityMap.ID;
+        const verdict = IntegrationEngine.ClassifyOversizedBatch(objectName, batch.Records.length, this.MaxBatchSize, batchIndex, batch.HasMore);
+        if (!verdict)
+            return;
+        logger?.warning(objectName, verdict.Code, verdict.Message, {
+            batchIndex,
+            recordCount: batch.Records.length,
+            requestedBatchSize: this.MaxBatchSize,
+            hasMore: batch.HasMore ?? null,
+            unbounded: verdict.Unbounded,
+        });
+        console.warn(`[IntegrationEngine] ${verdict.Message}`);
+    }
     /**
      * Push sync: detect local MJ record changes → reverse-map fields → push to external system.
      *
@@ -1924,7 +2500,19 @@ export class IntegrationEngine extends BaseSingleton {
         // lexicographically, matching the SQL `ChangedAt > '...'` filter in LoadChangedMJRecords.
         let firstErrorChangeAt = null; // min ChangedAt among failed pushes
         const successfulChangeAts = [];
+        let recordsSinceBoundary = 0;
         for (const change of changedRecords) {
+            // Push has no natural batch, so impose a fence boundary every MaxBatchSize records:
+            // a reclaimed run must stop writing to the VENDOR promptly, not at end-of-map. The
+            // check sits OUTSIDE the per-record try so RunOwnershipLostError propagates (the
+            // per-record catch must never swallow it). A cross-process cancel trips the abort
+            // signal, honored at the top of each iteration.
+            if (recordsSinceBoundary === 0)
+                await this.assertOwnershipAtBoundary();
+            if (++recordsSinceBoundary >= this.MaxBatchSize)
+                recordsSinceBoundary = 0;
+            if (_abortSignal?.aborted)
+                break;
             result.RecordsProcessed++;
             try {
                 await this.PushSingleRecord(change, config, entityMap, pushFieldMaps, result, contextUser, logger);
@@ -2515,6 +3103,8 @@ export class IntegrationEngine extends BaseSingleton {
                     result.RecordsSkipped += recs.length;
                     continue;
                 }
+                // Partition boundary = a batch boundary: fence-check before this partition's writes.
+                await this.assertOwnershipAtBoundary();
                 // D3: serialize the match READ through the same write-mutex the non-partition path uses
                 // (~line 1644). matchEngine.Resolve reads existing MJ rows on the SHARED provider
                 // connection, so when streams run in parallel (syncConcurrency>1) it must not interleave
@@ -3748,23 +4338,36 @@ export class IntegrationEngine extends BaseSingleton {
     async FinalizeRun(run, result, contextUser, onNotification, aborted) {
         run.EndedAt = new Date();
         run.TotalRecords = result.RecordsProcessed;
-        // A user/system-cancelled run must NOT be recorded as 'Success' — that hides the
-        // cancellation in run history (indistinguishable from a clean completion) and is wrong
-        // for any downstream cadence/health logic. Until a first-class 'Cancelled' status value
-        // exists on CompanyIntegrationRun (Status value list is Pending/In Progress/Success/Failed),
-        // finalize an aborted run as 'Failed' with an explicit ErrorLog. The durable progress
-        // artifact additionally carries exitReason='aborted' (see finalizeSyncProgress) so a stopped
-        // run stays distinguishable from a real failure over GraphQL.
+        // A cancelled run is neither a success nor a failure, and 'Cancelled' is now a first-class
+        // value in the Status list — so record it as itself. Previously this had to be 'Failed' with
+        // an explanatory ErrorLog, which meant every health/cadence consumer counted deliberate
+        // cancellations as errors unless it string-matched that text. The ErrorLog is still written
+        // (it carries the reason), and the durable progress artifact still carries
+        // exitReason='aborted' (see finalizeSyncProgress).
+        // Held in a local so the release below sends the SAME value. Deriving it there from
+        // `run.Status` with a two-way test collapsed everything non-Success to 'Failed', which would
+        // have overwritten 'Cancelled' in the release UPDATE — the sproc assigns Status = @FinalStatus,
+        // so the row's carefully-set status would be undone one statement later.
+        let terminalStatus;
         if (aborted) {
-            run.Status = 'Failed';
+            terminalStatus = 'Cancelled';
             run.ErrorLog = result.ErrorMessage ?? 'Sync cancelled by user';
         }
         else {
-            run.Status = result.RecordsErrored > 0 ? 'Failed' : 'Success';
+            terminalStatus = result.RecordsErrored > 0 ? 'Failed' : 'Success';
             if (result.Errors.length > 0) {
                 run.ErrorLog = JSON.stringify(result.Errors.slice(0, 100));
             }
         }
+        run.Status = terminalStatus;
+        // The generated spUpdate writes EVERY column from the entity's in-memory state.
+        // This run entity was loaded BEFORE the claim, so without a sync its ownership
+        // columns (FenceToken 0, OwnerToken null, stale lease) would clobber the DB's live
+        // values on save — silently un-fencing the run. Sync them to the service's
+        // last-known-authoritative values first; Release() below then clears ownership
+        // atomically (token-checked, so a stale holder's release no-ops).
+        const ownership = this.currentRunContext?.ownership;
+        ownership?.SyncEntityOwnershipFields(run);
         // Retry the finalize save: a failed save leaves the run 'In Progress', which ResumeOrphanedSyncs
         // re-queues on next startup → the whole sync re-runs (re-fetch + re-apply). Worth a few retries to
         // make the terminal status durable. Both a thrown infra error and a `false` logical-failure retry.
@@ -3781,6 +4384,16 @@ export class IntegrationEngine extends BaseSingleton {
                 `${saveErr instanceof Error ? saveErr.message : String(saveErr)}. ` +
                 `Run may remain 'In Progress' and be re-queued as orphaned on next startup.`);
         }
+        // Terminal release: clear OwnerToken/LeaseExpiresAt and (re-)stamp the final status
+        // in one token-checked statement, so the row is immediately claimable-clean.
+        if (ownership) {
+            try {
+                await ownership.Release(terminalStatus);
+            }
+            catch (releaseErr) {
+                console.warn(`[IntegrationEngine] Run ${run.ID} release failed (non-fatal — lease will simply expire): ${releaseErr instanceof Error ? releaseErr.message : String(releaseErr)}`);
+            }
+        }
         if (onNotification) {
             const notification = this.buildCompletionNotification(run, result);
             this.safeNotify(onNotification, notification);
@@ -3850,7 +4463,17 @@ export class IntegrationEngine extends BaseSingleton {
         run.EndedAt = new Date();
         run.Status = 'Failed';
         run.ErrorLog = err instanceof Error ? err.message : String(err);
+        // Same full-row-save hazard as FinalizeRun: sync the in-memory ownership columns to the
+        // claim's live values before Save, then release (token-checked — a fenced-out holder no-ops).
+        const ownership = this.currentRunContext?.ownership;
+        ownership?.SyncEntityOwnershipFields(run);
         await run.Save();
+        if (ownership) {
+            try {
+                await ownership.Release('Failed');
+            }
+            catch { /* non-fatal — lease will expire */ }
+        }
         if (onNotification) {
             const failResult = {
                 Success: false,
@@ -3968,7 +4591,7 @@ export class IntegrationEngine extends BaseSingleton {
                 CompanyIntegrationID: companyIntegrationID,
                 ContextUser: contextUser,
                 SyncedEntityNames: syncedEntityNames,
-                Provider: this._provider,
+                Provider: this.currentRunContext?.provider,
                 // The run's in-memory custom-key candidates — needed because the
                 // overflow-column scan alone under-reports once the hash basis excludes
                 // overflow (skipped rows never write their overflow JSON).