@skedulo/integration-sdk 0.11.3 → 0.12.0

package/dist/services/sync-runner.js

@@ -1,7 +1,19 @@
 "use strict";
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SyncRunner = exports.AdapterNotRegisteredError = exports.AdapterRoleError = void 0;
 exports.decideCursor = decideCursor;
+exports.syncScope = syncScope;
 const crypto_1 = require("crypto");
 const constants_1 = require("../constants");
 const integration_log_interface_1 = require("../interfaces/integration-log.interface");
@@ -13,6 +25,13 @@ const id_mapping_sync_strategy_1 = require("./id-mapping-sync.strategy");
 const id_mapping_service_1 = require("./id-mapping.service");
 const sync_service_1 = require("./sync.service");
 const helpers_1 = require("../utils/helpers");
+// The largest fields a hold or _countFailedResume can add later to a checkpoint that fit.
+const LATER_FIELDS = {
+    holds: 9999,
+    heldSince: new Date(0).toISOString(),
+    failedResumes: 9999,
+    failingSince: new Date(0).toISOString(),
+};
 const emptyBatchResult = () => ({
     succeeded: 0,
     filtered: 0,
@@ -51,10 +70,16 @@ class SyncRunner {
     // sourceSystem / targetSystem fallbacks are used when the flow's own values are null
     // (e.g. flows created before these columns were added to the schema).
     async run(args, ctx) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t;
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
         // One execution_id per run; correlates the run_summary + record_error logs.
         const executionId = (0, crypto_1.randomUUID)();
         const startedAt = Date.now();
+        let pendingResume = null;
+        let savedPages = 0;
+        let unsavedPages = 0;
+        let rowsWritten = 0;
+        let checkpointedFailed = 0;
+        let checkpointedSucceeded = 0;
         try {
             const flow = await ctx.dataFlowService.getFlowById(args.flowId);
             if (!flow) {
@@ -97,40 +122,291 @@ class SyncRunner {
             const lastRunAt = args.overrideSince !== undefined ? args.overrideSince : ((_f = flow.lastRunAt) !== null && _f !== void 0 ? _f : args.since);
             const overlapMs = (_h = (_g = args.options) === null || _g === void 0 ? void 0 : _g.cursorOverlapMs) !== null && _h !== void 0 ? _h : 60000;
             const effectiveCursor = lastRunAt ? new Date(lastRunAt.getTime() - overlapMs) : null; // null = full sync (no prior run)
-            const result = await new sync_service_1.SyncService(source, target, strategy).run({ flow, since: effectiveCursor }, ctx);
+            // Every run but a backfill saves progress. Of the backfills, only a full resync that finishes moves the cursor.
+            const resumable = typeof source.fetchResumable === 'function';
+            const backfill = args.overrideSince !== undefined;
+            const fullResync = args.overrideSince === null;
+            const canSave = resumable && !backfill;
+            const windowSince = (_j = effectiveCursor === null || effectiveCursor === void 0 ? void 0 : effectiveCursor.toISOString()) !== null && _j !== void 0 ? _j : null;
+            const scope = syncScope(flow);
+            const saved = flow.syncProgress;
+            let prior = canSave && (saved === null || saved === void 0 ? void 0 : saved.scope) === scope && saved.windowSince === windowSince ? saved : null;
+            if (canSave && saved && !prior) {
+                ctx.logger.warn(`sync_checkpoint_ignored: flowId=${args.flowId} changed=${saved.scope !== scope ? 'scope' : 'window'} — ` +
+                    `the flow or its cursor changed since the checkpoint was saved; starting the load over`);
+            }
+            // A saved checkpoint must fit a later hold; a future startedAt would move lastRunAt past the load's start.
+            if (prior &&
+                (!(Date.parse(prior.startedAt) <= runStartedAt.getTime()) ||
+                    !hasRoom(prior) ||
+                    [prior.processed, prior.holds, prior.failedResumes].some(notCount))) {
+                ctx.logger.warn(`sync_checkpoint_invalid: flowId=${args.flowId} startedAt="${prior.startedAt}" size=${JSON.stringify(prior).length} — starting the load over`);
+                prior = null;
+            }
+            const loadStartedAt = prior ? new Date(prior.startedAt) : runStartedAt;
+            if (prior && runStartedAt.getTime() - loadStartedAt.getTime() > constants_1.SLOW_LOAD_MS) {
+                ctx.logger.warn(`sync_load_slow: flowId=${args.flowId} loadStartedAt=${prior.startedAt} processed=${prior.processed} — ` +
+                    `this load started over ${constants_1.SLOW_LOAD_MS / 3600000} hours ago and hasn't finished; check its run logs for held pages or failed runs`);
+            }
+            pendingResume = (prior === null || prior === void 0 ? void 0 : prior.token) ? prior : null;
+            let last = prior;
+            let processed = (_k = prior === null || prior === void 0 ? void 0 : prior.processed) !== null && _k !== void 0 ? _k : 0;
+            let heldTries = 0;
+            let unrecordableAfterHolds = 0;
+            let unrecordableAtOnce = 0;
+            let tooLarge = false;
+            let droppedRows = 0;
+            const recordedErrors = new Set();
+            const countHeld = (reason) => ctx.metrics.counter('sync.checkpoint.held', 1, {
+                integrationType: ctx.integrationType,
+                reason,
+            });
+            const fits = (progress) => {
+                if (hasRoom(progress))
+                    return true;
+                const size = JSON.stringify(progress).length;
+                if (tooLarge)
+                    return false;
+                tooLarge = true;
+                ctx.logger.error(`sync_checkpoint_too_large: flowId=${args.flowId} size=${size} max=${constants_1.MAX_SYNC_PROGRESS_CHARS} — the adapter's resume token is too large to save; skipping the save for that page`);
+                ctx.metrics.counter('sync.checkpoint.too_large', 1, {
+                    integrationType: ctx.integrationType,
+                });
+                return false;
+            };
+            const capRows = (entries) => canSave ? entries : entries.slice(0, run_log_builder_1.MAX_RECORD_ERRORS - rowsWritten);
+            const saveProgress = async (progress) => {
+                await ctx.dataFlowService.patchCursor(args.flowId, { syncProgress: progress });
+                last = progress;
+                pendingResume = null;
+            };
+            const checkpoint = async (token, pageErrors, recordCount, succeeded, failed) => {
+                var _a, _b;
+                const entries = this._buildFailureEntries(ctx, flow, source, executionId, pageErrors);
+                const outage = looksLikeOutage(succeeded, entries);
+                const holdReason = entries.length < pageErrors.length || (failed > 0 && pageErrors.length === 0)
+                    ? 'unrecorded_failures'
+                    : outage
+                        ? 'outage_suspected'
+                        : undefined;
+                const progress = {
+                    token,
+                    windowSince,
+                    scope,
+                    startedAt: loadStartedAt.toISOString(),
+                    processed: processed + recordCount,
+                };
+                let skipped;
+                let skippedAfterHolds = false;
+                if (holdReason) {
+                    const { holds, heldSince, movePast } = nextHold(last);
+                    const held = Object.assign(Object.assign({}, progress), { token: (_a = last === null || last === void 0 ? void 0 : last.token) !== null && _a !== void 0 ? _a : null, processed: (_b = last === null || last === void 0 ? void 0 : last.processed) !== null && _b !== void 0 ? _b : 0, holds,
+                        heldSince });
+                    const saveHold = canSave && !movePast;
+                    // A run that can't save has no hold count, so only an outage stops it: waiting can't give a record an id.
+                    const stopReason = saveHold
+                        ? holdReason
+                        : !canSave && outage
+                            ? 'outage_suspected'
+                            : undefined;
+                    if (stopReason) {
+                        if (saveHold) {
+                            try {
+                                await saveProgress(held);
+                                heldTries = holds;
+                            }
+                            catch (err) {
+                                ctx.logger.error(`sync_checkpoint_hold_save_failed: flowId=${args.flowId} error="${err instanceof Error ? err.message : String(err)}"`);
+                                return 'checkpoint_held';
+                            }
+                        }
+                        ctx.logger.warn(`sync_checkpoint_held: flowId=${args.flowId} reason=${stopReason} ` +
+                            `errors=${pageErrors.length} recordable=${entries.length} ` +
+                            `${saveHold ? `holds=${holds}` : 'saves no progress'} — stopping on this page`);
+                        countHeld(stopReason);
+                        return 'checkpoint_held';
+                    }
+                    skipped = `reason=${holdReason} ${canSave ? `holds=${holds}` : 'saves no progress'}`;
+                    skippedAfterHolds = canSave;
+                }
+                const save = canSave && fits(progress);
+                const rows = capRows(entries);
+                const countFailed = () => ctx.metrics.counter('sync.checkpoint.failed', 1, {
+                    integrationType: ctx.integrationType,
+                });
+                try {
+                    if (rows.length > 0)
+                        await ctx.failedRecordClient.upsert(rows);
+                }
+                catch (err) {
+                    ctx.logger.error(`sync_checkpoint_failed: flowId=${args.flowId} error="${err instanceof Error ? err.message : String(err)}" — failed records not saved, stopping; the next run resumes from the previous checkpoint`);
+                    countFailed();
+                    return 'checkpoint_failed';
+                }
+                if (save) {
+                    try {
+                        await saveProgress(progress);
+                        savedPages++;
+                        unsavedPages = 0;
+                    }
+                    catch (err) {
+                        ctx.logger.warn(`sync_progress_save_failed: flowId=${args.flowId} error="${err instanceof Error ? err.message : String(err)}" — still reading; the next run may re-read this page`);
+                        countFailed();
+                        unsavedPages++;
+                        // The stored token's holds belong to an earlier page; the next held page starts its own count.
+                        if (last)
+                            last = Object.assign(Object.assign({}, last), { holds: undefined, heldSince: undefined });
+                    }
+                }
+                rowsWritten += rows.length;
+                droppedRows += entries.length - rows.length;
+                if (skipped) {
+                    const unrecordable = pageErrors.length - entries.length;
+                    if (skippedAfterHolds)
+                        unrecordableAfterHolds += unrecordable;
+                    else
+                        unrecordableAtOnce += unrecordable;
+                    ctx.logger.warn(`sync_checkpoint_hold_skipped: flowId=${args.flowId} ${skipped} ` +
+                        `recorded=${rows.length} unrecordable=${unrecordable} — moving past the page`);
+                }
+                if (rows.length > 0) {
+                    ctx.metrics.counter('sync.failures.recorded', rows.length, {
+                        integrationType: ctx.integrationType,
+                    });
+                }
+                // Only the first MAX_RECORD_ERRORS can be in result.errors, so that's all the finish needs.
+                for (const e of pageErrors) {
+                    if (recordedErrors.size >= run_log_builder_1.MAX_RECORD_ERRORS)
+                        break;
+                    recordedErrors.add(e);
+                }
+                checkpointedFailed += failed;
+                checkpointedSucceeded += succeeded;
+                processed = progress.processed;
+                return undefined;
+            };
+            const result = await new sync_service_1.SyncService(source, target, strategy).run(Object.assign({ flow, since: effectiveCursor }, (resumable
+                ? {
+                    resume: {
+                        token: (_l = prior === null || prior === void 0 ? void 0 : prior.token) !== null && _l !== void 0 ? _l : null,
+                        deadlineAt: (_m = args.options) === null || _m === void 0 ? void 0 : _m.deadlineAt,
+                        checkpoint,
+                    },
+                }
+                : {})), ctx);
+            if (pendingResume === null || pendingResume === void 0 ? void 0 : pendingResume.failedResumes) {
+                try {
+                    await saveProgress(Object.assign(Object.assign({}, pendingResume), { failedResumes: undefined, failingSince: undefined }));
+                }
+                catch (err) {
+                    ctx.logger.error(`sync_failed_resume_clear_failed: flowId=${args.flowId} error="${err instanceof Error ? err.message : String(err)}"`);
+                }
+            }
+            const pending = result.stoppedEarly
+                ? []
+                : ((_o = result.errors) !== null && _o !== void 0 ? _o : []).filter((e) => !recordedErrors.has(e));
+            const failures = this._buildFailureEntries(ctx, flow, source, executionId, pending);
+            // A run that saves progress never drops a failure; the others keep the 0.11.x cap.
+            const reportDropped = () => {
+                var _a, _b, _c, _d, _e;
+                const dropped = resumable
+                    ? droppedRows
+                    : ((_a = result.errorCount) !== null && _a !== void 0 ? _a : 0) - ((_c = (_b = result.errors) === null || _b === void 0 ? void 0 : _b.length) !== null && _c !== void 0 ? _c : 0);
+                if (dropped <= 0)
+                    return undefined;
+                ctx.metrics.counter('sync.failures.dropped', dropped, {
+                    integrationType: ctx.integrationType,
+                });
+                ctx.logger.warn(`failed_records_truncated: flowId=${args.flowId} recorded=${resumable ? rowsWritten : ((_e = (_d = result.errors) === null || _d === void 0 ? void 0 : _d.length) !== null && _e !== void 0 ? _e : 0)} ` +
+                    `dropped=${dropped} — ` +
+                    (resumable
+                        ? `a backfill or full resync records at most ${run_log_builder_1.MAX_RECORD_ERRORS} failures; retry the recorded ones, then re-run it to record the rest`
+                        : 'replay the window (run-flow --since) to recover the remainder'));
+                return (0, run_log_builder_1.droppedNote)(dropped);
+            };
+            // Failures were saved at each checkpoint. The cursor moves only when the load finishes.
+            if (result.stoppedEarly) {
+                const note = joinNotes((0, run_log_builder_1.stoppedEarlyNote)({
+                    stoppedEarly: result.stoppedEarly,
+                    backfill,
+                    fullResync,
+                    tooLarge,
+                    saved: Boolean(last === null || last === void 0 ? void 0 : last.token),
+                    heldTries,
+                }), (0, run_log_builder_1.unrecordableNote)(unrecordableAfterHolds, unrecordableAtOnce), reportDropped(), unsavedPages > 0
+                    ? `saving progress failed on ${unsavedPages} page${unsavedPages === 1 ? ', the next run may re-read it' : 's, the next run may re-read them'}`
+                    : undefined);
+                ctx.logger.info(`sync_stopped_early: flowId=${args.flowId} reason=${result.stoppedEarly} processed=${processed} ` +
+                    `loadStartedAt=${loadStartedAt.toISOString()} — ${note}`);
+                ctx.metrics.counter('sync.run.stopped_early', 1, {
+                    integrationType: ctx.integrationType,
+                    reason: result.stoppedEarly,
+                });
+                await this._emitRunLog(ctx, Object.assign(Object.assign({ executionId, flowId: args.flowId, flowName: flow.name, startedAt, summary: result }, (unsavedPages > 0 && result.failed === 0 ? { summaryLevel: integration_log_interface_1.LogLevel.WARNING } : {})), { windowFrom: effectiveCursor, windowTo: loadStartedAt, note }));
+                return result;
+            }
             // Record each failed record, then always advance — one bad record must not wedge the
             // flow (SOL-1198). We only advance once the failures are safely written: if the write
             // throws, a retained error has no source id to record, or the run looks like an outage,
             // hold the cursor and let the next run redo the window.
-            const failures = this._buildFailureEntries(ctx, flow, source, executionId, (_j = result.errors) !== null && _j !== void 0 ? _j : []);
-            // Deliberate limitation: the retained-errors cap (MAX_RECORD_ERRORS) bounds how many rows a
-            // mass-failure run dead-letters, so the overflow is NOT written and the cursor still advances
-            // past it. That window is replay-recoverable (run-flow --since), not lost for good, but it
-            // does not auto-recover — surface the dropped count (metric + warn) so it isn't silent.
-            const dropped = ((_k = result.errorCount) !== null && _k !== void 0 ? _k : 0) - ((_m = (_l = result.errors) === null || _l === void 0 ? void 0 : _l.length) !== null && _m !== void 0 ? _m : 0);
-            if (dropped > 0) {
-                ctx.metrics.counter('sync.failures.dropped', dropped, {
-                    integrationType: ctx.integrationType,
-                });
-                ctx.logger.warn(`failed_records_truncated: flowId=${args.flowId} recorded=${(_p = (_o = result.errors) === null || _o === void 0 ? void 0 : _o.length) !== null && _p !== void 0 ? _p : 0} ` +
-                    `dropped=${dropped} — replay the window (run-flow --since) to recover the remainder`);
+            const lastPageSucceeded = result.succeeded - checkpointedSucceeded;
+            const decision = decideCursor(Object.assign(Object.assign({}, result), { succeeded: lastPageSucceeded, failed: result.failed - checkpointedFailed, errors: pending }), failures);
+            // Same as a mid page of a run that can't save: an outage wins over a record with no source id.
+            if (resumable &&
+                !canSave &&
+                decision.reason === 'unrecorded_failures' &&
+                looksLikeOutage(lastPageSucceeded, failures)) {
+                decision.reason = 'outage_suspected';
+            }
+            let advance = decision.advance;
+            let lastPageHolds = 0;
+            if (!advance && !result.holdCursor && last) {
+                const { holds, heldSince, movePast } = nextHold(last);
+                if (movePast) {
+                    advance = true;
+                    lastPageHolds = holds;
+                }
+                else {
+                    const held = Object.assign(Object.assign({}, last), { holds,
+                        heldSince, failedResumes: undefined, failingSince: undefined });
+                    try {
+                        await saveProgress(held);
+                        heldTries = holds;
+                        countHeld(decision.reason);
+                    }
+                    catch (err) {
+                        ctx.logger.error(`sync_checkpoint_hold_save_failed: flowId=${args.flowId} error="${err instanceof Error ? err.message : String(err)}"`);
+                    }
+                }
             }
-            const decision = decideCursor(result, failures);
             // Failures become rows when advancing (fail-closed gate: advancing past an unrecorded
             // failure would be silent data loss) AND during delete-window holds (result.holdCursor)
             // — a delete hold can persist across many runs, and hiding the run's other failures
             // from the Activity drill-in for that long re-creates the COR-818 wedge. Outage and
-            // unrecorded holds stay row-free as designed (transient flood / unattributable errors).
+            // unrecorded holds stay row-free as designed (transient flood / unattributable errors),
+            // except an unrecorded hold in a resumable run that saves no progress: its mid pages
+            // record what they can, and a re-run can't give a record a source id.
             // Rows are keyed (tenant, flow, source_id), so re-recording each held re-run is idempotent.
             let recorded = true;
-            if (decision.advance || result.holdCursor) {
+            const recordHeld = resumable && !canSave && decision.reason === 'unrecorded_failures';
+            const rows = capRows(failures);
+            let rowsRecorded = 0;
+            if (advance || result.holdCursor || recordHeld) {
                 try {
-                    await ctx.failedRecordClient.upsert(failures);
-                    if (failures.length > 0) {
-                        ctx.metrics.counter('sync.failures.recorded', failures.length, {
+                    await ctx.failedRecordClient.upsert(rows);
+                    droppedRows += failures.length - rows.length;
+                    rowsWritten += rows.length;
+                    rowsRecorded = rows.length;
+                    if (rows.length > 0) {
+                        ctx.metrics.counter('sync.failures.recorded', rows.length, {
                             integrationType: ctx.integrationType,
                         });
                     }
+                    if (lastPageHolds > 0) {
+                        unrecordableAfterHolds += pending.length - failures.length;
+                        ctx.logger.warn(`sync_checkpoint_hold_skipped: flowId=${args.flowId} reason=${decision.reason} holds=${lastPageHolds} ` +
+                            `recorded=${rows.length} unrecordable=${pending.length - failures.length} — moving past the last page`);
+                    }
                 }
                 catch (err) {
                     recorded = false;
@@ -140,28 +416,55 @@ class SyncRunner {
                     });
                 }
             }
-            if (decision.advance && recorded) {
-                try {
-                    await ctx.dataFlowService.updateCursor(args.flowId, runStartedAt);
+            // Use the load's start so changes made during the load are picked up next run.
+            const cursor = loadStartedAt;
+            const keptBy = !backfill
+                ? undefined
+                : !fullResync
+                    ? 'backfill does not move the cursor'
+                    : droppedRows > 0 || unrecordableAtOnce > 0
+                        ? 'full resync did not record every failure, so it does not move the cursor'
+                        : undefined;
+            let note = keptBy;
+            if (advance && recorded) {
+                if (keptBy) {
+                    ctx.logger.info(`sync_cursor_kept: flowId=${args.flowId} — ${keptBy} or touch the checkpoint`);
                 }
-                catch (err) {
-                    // Failures ARE recorded; only the cursor didn't persist. Idempotent upserts make
-                    // re-processing the window next run harmless.
-                    ctx.logger.error(`update_cursor_failed: flowId=${args.flowId} error="${err instanceof Error ? err.message : String(err)}" — next run will re-process same window`);
+                else {
+                    try {
+                        // A fullResync read every record, so it may also clear a scheduled load's checkpoint.
+                        await ctx.dataFlowService.patchCursor(args.flowId, Object.assign({ lastRunAt: cursor }, (savedPages > 0 || flow.syncProgress ? { syncProgress: null } : {})));
+                    }
+                    catch (err) {
+                        // Failures ARE recorded; only the cursor didn't persist. Idempotent upserts make
+                        // re-processing the window next run harmless.
+                        ctx.logger.error(`update_cursor_failed: flowId=${args.flowId} error="${err instanceof Error ? err.message : String(err)}" — next run will re-process same window`);
+                    }
                 }
             }
             else {
                 // Stamp the hold reason on the summary so the tenant-facing run log says the
                 // window will repeat — not just the log line and metric.
-                const reason = (_q = decision.reason) !== null && _q !== void 0 ? _q : 'failed_record_write_failed';
-                (_r = result.holdCursor) !== null && _r !== void 0 ? _r : (result.holdCursor = reason);
+                const reason = advance
+                    ? 'failed_record_write_failed'
+                    : ((_p = decision.reason) !== null && _p !== void 0 ? _p : 'failed_record_write_failed');
+                (_q = result.holdCursor) !== null && _q !== void 0 ? _q : (result.holdCursor = reason);
+                note = (0, run_log_builder_1.heldCursorNote)({
+                    reason,
+                    backfill,
+                    fullResync,
+                    heldTries,
+                    rowsSaved: recordHeld && recorded && rows.length > 0,
+                    writeFailed: recordHeld && !recorded,
+                });
                 ctx.logger.warn(`cursor_held: flowId=${args.flowId} reason=${reason} failed=${result.failed} ` +
-                    `recorded=${failures.length} retained=${(_t = (_s = result.errors) === null || _s === void 0 ? void 0 : _s.length) !== null && _t !== void 0 ? _t : 0} — window retries next run`);
+                    `recorded=${rowsRecorded} retained=${(_s = (_r = result.errors) === null || _r === void 0 ? void 0 : _r.length) !== null && _s !== void 0 ? _s : 0} — ${note}`);
                 ctx.metrics.counter('sync.cursor.held', 1, {
                     integrationType: ctx.integrationType,
                     reason,
                 });
             }
+            note = joinNotes(note, (0, run_log_builder_1.unrecordableNote)(unrecordableAfterHolds, unrecordableAtOnce), reportDropped());
             await this._emitRunLog(ctx, {
                 executionId,
                 flowId: args.flowId,
@@ -169,7 +472,8 @@ class SyncRunner {
                 startedAt,
                 summary: result,
                 windowFrom: effectiveCursor,
-                windowTo: runStartedAt,
+                windowTo: cursor,
+                note,
             });
             return result;
         }
@@ -178,11 +482,14 @@ class SyncRunner {
             // then either swallow adapter-config errors (return) or re-throw unexpected ones.
             const msg = err instanceof Error ? err.message : String(err);
             ctx.logger.error(`sync_runner_run: ${msg} flowId=${args.flowId}`);
+            if (pendingResume && err instanceof sync_service_1.ResumeTokenError) {
+                await this._countFailedResume(ctx, args.flowId, pendingResume);
+            }
             const failedResult = {
-                succeeded: 0,
+                succeeded: checkpointedSucceeded,
                 filtered: 0,
                 skipped: 0,
-                failed: 1,
+                failed: checkpointedFailed + 1,
                 dlqWrites: 0,
             };
             await this._emitRunLog(ctx, {
@@ -190,7 +497,7 @@ class SyncRunner {
                 flowId: args.flowId,
                 startedAt,
                 summary: failedResult,
-                summaryMessage: `Run failed: ${msg}`,
+                summaryMessage: `Run failed: ${msg}${savedBeforeFailure(savedPages, rowsWritten, checkpointedSucceeded)}`,
             });
             if (err instanceof AdapterNotRegisteredError || err instanceof AdapterRoleError) {
                 return failedResult;
@@ -198,33 +505,47 @@ class SyncRunner {
             throw err;
         }
     }
+    // A token the adapter keeps failing on would fail every run; the 1-hour minimum keeps a short outage from dropping it.
+    async _countFailedResume(ctx, flowId, resumed) {
+        var _a;
+        const now = Date.now();
+        const failedResumes = ((_a = resumed.failedResumes) !== null && _a !== void 0 ? _a : 0) + 1;
+        const firstFailure = pastOrNow(resumed.failingSince, now);
+        const failingSince = new Date(firstFailure).toISOString();
+        const drop = failedResumes >= constants_1.MAX_FAILED_RESUMES && now - firstFailure >= constants_1.FAILED_RESUME_MIN_MS;
+        try {
+            await ctx.dataFlowService.patchCursor(flowId, {
+                syncProgress: drop
+                    ? {
+                        token: null,
+                        windowSince: resumed.windowSince,
+                        scope: resumed.scope,
+                        startedAt: resumed.startedAt,
+                        processed: 0,
+                    }
+                    : Object.assign(Object.assign({}, resumed), { failedResumes, failingSince }),
+            });
+        }
+        catch (err) {
+            ctx.logger.error(`sync_failed_resume_save_failed: flowId=${flowId} failedResumes=${failedResumes} error="${err instanceof Error ? err.message : String(err)}" — the failed resume isn't counted`);
+            return;
+        }
+        if (drop) {
+            ctx.logger.warn(`sync_checkpoint_dropped: flowId=${flowId} failedResumes=${failedResumes} failingSince=${failingSince} — the first page after the saved token keeps failing to load; the next run reads from the start, keeping the load's startedAt`);
+        }
+        else {
+            ctx.logger.warn(`sync_checkpoint_failed_resume: flowId=${flowId} failedResumes=${failedResumes} failingSince=${failingSince} — ` +
+                `the checkpoint drops after ${constants_1.MAX_FAILED_RESUMES} failed resumes over at least ${constants_1.FAILED_RESUME_MIN_MS / 60000} min`);
+        }
+    }
     // Build and flush this run's logs. Failures are swallowed + logged — logging
     // must never break a sync. source defaults to scheduled_job when part of a job,
     // else batch; the event paths pass 'event' explicitly.
     async _emitRunLog(ctx, args) {
-        var _a, _b;
-        const entries = (0, run_log_builder_1.buildRunLogEntries)({
-            integrationType: ctx.integrationType,
-            flowId: args.flowId,
-            flowName: (_a = args.flowName) !== null && _a !== void 0 ? _a : null,
-            executionId: args.executionId,
-            source: (_b = args.source) !== null && _b !== void 0 ? _b : (ctx.jobRunId ? integration_log_interface_1.LogSource.SCHEDULED_JOB : integration_log_interface_1.LogSource.BATCH),
-            durationMs: Date.now() - args.startedAt,
-            summary: args.summary,
-            errors: args.summary.errors,
-            errorCount: args.summary.errorCount,
-            jobRunId: ctx.jobRunId,
-            windowFrom: args.windowFrom,
-            windowTo: args.windowTo,
-            summaryLevel: args.summaryLevel,
-            summaryMessage: args.summaryMessage,
-        });
-        await ctx.integrationLogClient.create(entries).catch((err) => {
-            // A dropped flush would hide the very failures this feature exists to surface —
-            // emit a metric so the silent drop is observable, not just a stdout line.
-            ctx.metrics.counter('run_log.flush_failed', 1, { integrationType: ctx.integrationType });
-            ctx.logger.error(`run_log_flush_failed: flowId=${args.flowId} error="${err instanceof Error ? err.message : String(err)}"`);
-        });
+        var _a;
+        const { startedAt } = args, rest = __rest(args, ["startedAt"]);
+        const entries = (0, run_log_builder_1.buildRunLogEntries)(Object.assign(Object.assign({}, rest), { integrationType: ctx.integrationType, source: (_a = args.source) !== null && _a !== void 0 ? _a : (ctx.jobRunId ? integration_log_interface_1.LogSource.SCHEDULED_JOB : integration_log_interface_1.LogSource.BATCH), durationMs: Date.now() - startedAt, errors: args.summary.errors, errorCount: args.summary.errorCount, jobRunId: ctx.jobRunId }));
+        await (0, run_log_builder_1.flushRunLog)(ctx, entries, args.flowId);
     }
     // Inbound (vendor→Skedulo) batch path. Fans one delivery out to every flow
     // matching the changed object, but processes each flow's share as ONE batch
@@ -593,13 +914,66 @@ function decideCursor(result, failures) {
     if (retained === 0 || failures.length < retained) {
         return { advance: false, reason: 'unrecorded_failures' };
     }
-    if (result.succeeded === 0 &&
-        failures.length > 0 &&
-        failures.every((f) => f.errorClass === 'transient')) {
+    if (looksLikeOutage(result.succeeded, failures)) {
         return { advance: false, reason: 'outage_suspected' };
     }
     return { advance: true };
 }
+// Every hold gets the 1-hour minimum: an outage page with one id-less record is held as unrecorded_failures.
+function nextHold(last) {
+    var _a;
+    const now = Date.now();
+    const holds = ((_a = last === null || last === void 0 ? void 0 : last.holds) !== null && _a !== void 0 ? _a : 0) + 1;
+    const since = pastOrNow(last === null || last === void 0 ? void 0 : last.heldSince, now);
+    const movePast = holds >= constants_1.MAX_CHECKPOINT_HOLDS && now - since >= constants_1.FAILED_RESUME_MIN_MS;
+    return { holds, heldSince: new Date(since).toISOString(), movePast };
+}
+// A future or unparsable time (clock skew, bad data) counts as now, or it would never be old enough.
+function pastOrNow(iso, now) {
+    return Math.min(Date.parse(iso !== null && iso !== void 0 ? iso : '') || now, now);
+}
+function savedBeforeFailure(pages, rows, synced) {
+    if (pages === 0 && rows === 0)
+        return '';
+    const saved = pages > 0
+        ? `saved ${pages} page${pages === 1 ? '' : 's'} (${synced} synced), the next run continues from the last one`
+        : 'saved no progress';
+    return ` — before it failed the run ${saved}; ${rows} failed record${rows === 1 ? '' : 's'} recorded`;
+}
+function notCount(value) {
+    return value !== undefined && !(Number.isInteger(value) && Number(value) >= 0);
+}
+function hasRoom(progress) {
+    return JSON.stringify(Object.assign(Object.assign({}, progress), LATER_FIELDS)).length <= constants_1.MAX_SYNC_PROGRESS_CHARS;
+}
+function joinNotes(...notes) {
+    return notes.filter(Boolean).join('; ') || undefined;
+}
+function looksLikeOutage(succeeded, failures) {
+    return (succeeded === 0 && failures.length > 0 && failures.every((f) => f.errorClass === 'transient'));
+}
+function syncScope(flow) {
+    var _a;
+    const fields = [
+        flow.sourceObject,
+        flow.targetObject,
+        (_a = flow.filter) !== null && _a !== void 0 ? _a : null,
+        flow.mappingConfig,
+        flow.matchKeyField,
+    ];
+    return (0, crypto_1.createHash)('sha256').update(stableJson(fields)).digest('hex').slice(0, 16);
+}
+function stableJson(value) {
+    var _a;
+    if (Array.isArray(value))
+        return `[${value.map(stableJson).join(',')}]`;
+    if (value && typeof value === 'object') {
+        const obj = value;
+        const keys = Object.keys(obj).sort();
+        return `{${keys.map((k) => `${JSON.stringify(k)}:${stableJson(obj[k])}`).join(',')}}`;
+    }
+    return (_a = JSON.stringify(value)) !== null && _a !== void 0 ? _a : 'null';
+}
 function createSyncStrategy(options, _ctx) {
     if (options.lookupStrategy === constants_1.LookupStrategy.IdMapping) {
         return new id_mapping_sync_strategy_1.IdMappingSyncStrategy(new id_mapping_service_1.IdMappingService());