meadow-integration 1.0.41 → 1.0.42

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "meadow-integration",
-    "version": "1.0.41",
+    "version": "1.0.42",
     "description": "Meadow Data Integration",
     "retoldBeacon": {
         "displayName": "Meadow Integration",
@@ -59,8 +59,8 @@
     "license": "MIT",
     "devDependencies": {
         "meadow-connection-sqlite": "^1.0.20",
-        "pict-docuserve": "^1.3.2",
-        "quackage": "^1.2.3"
+        "pict-docuserve": "^1.4.19",
+        "quackage": "^1.3.0"
     },
     "mocha": {
         "diff": true,
@@ -84,15 +84,15 @@
         "fable": "^3.1.75",
         "fable-serviceproviderbase": "^3.0.19",
         "fast-xml-parser": "^4.4.1",
-        "meadow": "^2.0.41",
+        "meadow": "^2.0.43",
         "meadow-connection-mssql": "^1.0.23",
         "meadow-connection-mysql": "^1.0.19",
         "orator": "^6.1.2",
         "orator-serviceserver-restify": "^2.0.11",
-        "pict-provider-theme": "^1.0.1",
+        "pict-provider-theme": "^1.1.2",
         "pict-section-flow": "^1.0.1",
-        "pict-section-modal": "^1.1.1",
-        "pict-section-theme": "^1.0.5",
+        "pict-section-modal": "^1.1.4",
+        "pict-section-theme": "^1.1.1",
         "pict-service-commandlineutility": "^1.0.19",
         "pict-sessionmanager": "^1.0.2",
         "pict-view": "^1.0.68",

package/source/services/clone/Meadow-Service-DeleteCursorStore.js

@@ -0,0 +1,105 @@
+/**
+ * Meadow Delete Cursor Store
+ *
+ * Tiny JSON-file store for the resumable delete-sync cursor.  One file holds a
+ * map keyed by table name; each entry is a few numbers (head/tail marks + a
+ * caught-up flag + a last-sweep timestamp).  Deliberately NOT a database: the
+ * payload is trivial, there is a single writer (the headless run is sequential),
+ * and a human-readable file is easy to inspect and mount on a volume.
+ *
+ * The file must live on a path that survives between runs/containers for the
+ * cursor to actually advance; if it doesn't, every load returns empty and the
+ * delete sync simply falls back to a full sweep (safe, just not incremental).
+ *
+ * All operations are best-effort and never throw to the caller: a missing or
+ * corrupt file reads as "no state" so the sync degrades to a full sweep rather
+ * than failing.  Writes are atomic (temp file + rename) and merge-preserve other
+ * tables' entries.
+ *
+ * @typedef {Object} DeleteCursorState
+ * @property {number} HeadID        Highest deleted id already covered from the top (0 = not yet established).
+ * @property {?number} TailID       Resume point for the downward catch-up sweep (null = sweep from the top).
+ * @property {boolean} CaughtUp     True once the tail has drained to the bottom of the deleted set.
+ * @property {number} LastSweepEpoch Epoch ms when the last full sweep completed (for the re-sweep cadence).
+ */
+const libFS = require('fs');
+const libPath = require('path');
+
+class MeadowDeleteCursorStore
+{
+	/**
+	 * @param {string} pStatePath - Filesystem path to the JSON state file.
+	 * @param {Object} [pLog] - Optional logger ({ warn }) for surfacing write failures.
+	 */
+	constructor(pStatePath, pLog)
+	{
+		this.statePath = pStatePath;
+		this.log = pLog || null;
+	}
+
+	/**
+	 * Read the entire state map. Missing/corrupt file → {} (graceful).
+	 * @return {Object<string, DeleteCursorState>}
+	 */
+	readAll()
+	{
+		try
+		{
+			const tmpRaw = libFS.readFileSync(this.statePath, 'utf8');
+			const tmpParsed = JSON.parse(tmpRaw);
+			return (tmpParsed && typeof(tmpParsed) === 'object') ? tmpParsed : {};
+		}
+		catch (pError)
+		{
+			// ENOENT (first run) and parse errors both degrade to "no state".
+			if (this.log && pError && pError.code !== 'ENOENT')
+			{
+				this.log.warn(`Delete cursor state unreadable at [${this.statePath}] (${pError.message}); treating as empty (full sweep).`);
+			}
+			return {};
+		}
+	}
+
+	/**
+	 * Get one table's cursor state, or null if none recorded.
+	 * @param {string} pTableName
+	 * @return {?DeleteCursorState}
+	 */
+	get(pTableName)
+	{
+		const tmpAll = this.readAll();
+		return Object.prototype.hasOwnProperty.call(tmpAll, pTableName) ? tmpAll[pTableName] : null;
+	}
+
+	/**
+	 * Persist one table's cursor state, preserving all other tables' entries.
+	 * Atomic (temp + rename). Returns true on success, false on failure (never throws).
+	 * @param {string} pTableName
+	 * @param {DeleteCursorState} pState
+	 * @return {boolean}
+	 */
+	set(pTableName, pState)
+	{
+		try
+		{
+			const tmpAll = this.readAll();
+			tmpAll[pTableName] = pState;
+
+			libFS.mkdirSync(libPath.dirname(this.statePath), { recursive: true });
+			const tmpTempPath = `${this.statePath}.${process.pid}.tmp`;
+			libFS.writeFileSync(tmpTempPath, JSON.stringify(tmpAll, null, '\t'), 'utf8');
+			libFS.renameSync(tmpTempPath, this.statePath);
+			return true;
+		}
+		catch (pError)
+		{
+			if (this.log)
+			{
+				this.log.warn(`Could not persist delete cursor for [${pTableName}] at [${this.statePath}] (${pError && pError.message}); progress will not carry to the next run.`);
+			}
+			return false;
+		}
+	}
+}
+
+module.exports = MeadowDeleteCursorStore;

package/source/services/clone/Meadow-Service-Sync-Entity-OngoingEventualConsistency.js

@@ -1,4 +1,5 @@
 const libMeadowSyncEntityOngoing = require('./Meadow-Service-Sync-Entity-Ongoing.js');
+const libMeadowDeleteCursorStore = require('./Meadow-Service-DeleteCursorStore.js');
 
 class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngoing
 {
@@ -13,6 +14,19 @@ class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngo
 		this.BackSyncTimeLimit = (typeof(this.options.BackSyncTimeLimit) === 'number')
 			? this.options.BackSyncTimeLimit
 			: 30000;
+
+		// Optional resumable-delete cursor.  When a state-file path is configured
+		// (via options or fable settings), delete reconciliation persists how far
+		// it has progressed so each run resumes instead of re-walking from the top.
+		// Unset → disabled, and delete sync behaves exactly as the non-cursor path.
+		// The file must live on a path that survives between runs to be useful.
+		const tmpSettings = this.fable.settings || {};
+		this.DeleteCursorStatePath = this.options.DeleteCursorStatePath || tmpSettings.DeleteCursorStatePath || '';
+		// Hours between full re-sweeps once caught up (catches deletions of older
+		// records that landed in already-swept id ranges).  Default 1 week.
+		this.DeleteResweepIntervalHours = (typeof(this.options.DeleteResweepIntervalHours) === 'number')
+			? this.options.DeleteResweepIntervalHours
+			: (typeof(tmpSettings.DeleteResweepIntervalHours) === 'number') ? tmpSettings.DeleteResweepIntervalHours : 168;
 	}
 
 	// Bisect an ID range with a time budget.  Checks the budget at the start of
@@ -139,9 +153,29 @@ class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngo
 	}
 
 	// Override deleted record sync with a time-budgeted version.
-	// The base syncDeletedRecords() walks ALL deleted records on the server
-	// with no limit, which defeats the purpose of eventual consistency.
-	// This version processes pages until BackSyncTimeLimit is exhausted.
+	//
+	// The server does not return deleted rows through the normal endpoints, so
+	// we page the deleted set explicitly and flag the matching LOCAL row deleted.
+	//
+	// SAFETY — match on IDENTITY, never on GUID.  This is a database clone: the
+	// identity column is authoritative and equals the origin's (joins and FKs
+	// depend on it), while GUIDs are NOT guaranteed unique in this data.  Acting
+	// on a GUID match could soft-delete the WRONG row (a different, possibly
+	// active, record that happens to share the GUID).  So we read and delete only
+	// by the row whose id equals the server's deleted row's id.
+	//
+	// A deleted server row whose id is not present locally was deleted before it
+	// was ever cloned here.  We do NOT create it: with duplicate GUIDs a create
+	// collides on the GUID unique index (the old "...already exists!" log storm),
+	// and backfilling those rows + de-duplicating GUIDs is a separate cleanup.
+	// We count them so the remaining backlog is visible.
+	//
+	// Ordering: newest-first by the indexed identity column.  Sorting by
+	// DeleteDate is honored by the API but is an unindexed filesort (10-150x
+	// slower on large tables — Observation: 87s for one page), so we order by the
+	// PK and stay within budget.  DeleteDate is present in the payload, so a
+	// future steady-state cursor can track the server's delete high-water mark
+	// without paying for the sort.  Processing is time-budgeted to BackSyncTimeLimit.
 	syncDeletedRecords(fCallback)
 	{
 		const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
@@ -151,7 +185,15 @@ class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngo
 			return fCallback();
 		}
 
-		this.fable.log.info(`Checking for deleted records on server for ${this.EntitySchema.TableName} (time-budgeted)...`);
+		// Opt-in resumable cursor: when a state-file path is configured, persist
+		// how far reconciliation has progressed so each run resumes instead of
+		// re-walking from the newest record every time.  Unset → this exact path.
+		if (this.DeleteCursorStatePath)
+		{
+			return this._syncDeletedRecordsWithCursor(fCallback);
+		}
+
+		this.fable.log.info(`Checking for deleted records on server for ${this.EntitySchema.TableName} (time-budgeted, matching on ${this.DefaultIdentifier})...`);
 
 		this.fable.MeadowCloneRestClient.getJSON(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`),
 			(pError, pResponse, pBody) =>
@@ -169,122 +211,66 @@ class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngo
 					return fCallback();
 				}
 
-				this.fable.log.info(`Found ${tmpDeletedCount} deleted records on server for ${this.EntitySchema.TableName}; syncing deletions with ${this.BackSyncTimeLimit}ms budget...`);
-
-				let tmpDeleteCap = (this.MaxRecordsPerEntity > 0)
+				const tmpDeleteCap = (this.MaxRecordsPerEntity > 0)
 					? Math.min(tmpDeletedCount, this.MaxRecordsPerEntity)
 					: tmpDeletedCount;
 
+				this.fable.log.info(`Found ${tmpDeletedCount} deleted records on server for ${this.EntitySchema.TableName}; reconciling newest-first with ${this.BackSyncTimeLimit}ms budget...`);
+
 				const tmpStartTime = Date.now();
-				let tmpProcessed = 0;
 				let tmpOffset = 0;
+				const tmpCounters = { seen: 0, marked: 0, already: 0, notInClone: 0, errors: 0 };
+
+				const fFinish = (pReason) =>
+				{
+					const tmpElapsed = Date.now() - tmpStartTime;
+					// Surface the real delete count in the structured run report
+					// (it was previously hard-coded to 0).
+					if (this.syncResults)
+					{
+						this.syncResults.Deleted = tmpCounters.marked;
+					}
+					this.fable.log.info(`Delete sync ${pReason} for ${this.EntitySchema.TableName} after ${tmpElapsed}ms: marked ${tmpCounters.marked}, already-deleted ${tmpCounters.already}, not-in-clone ${tmpCounters.notInClone}, errors ${tmpCounters.errors} (examined ${tmpCounters.seen} of ${tmpDeletedCount}).`);
+					return fCallback();
+				};
 
 				// Fetch deleted record pages one at a time using a recursive
-				// fetcher instead of pre-generating all URL partials.  With
-				// millions of deleted records and a short time budget, the old
-				// eachLimit approach generated hundreds of thousands of partials
-				// and then had to skip them all when the budget expired.
+				// fetcher.  Newest-first by the indexed identity column so recent
+				// deletions are reconciled within budget even when the historical
+				// backlog is large.
 				const fFetchDeletedPage = () =>
 				{
-					// Time budget check — stop immediately
 					if (Date.now() - tmpStartTime >= this.BackSyncTimeLimit)
 					{
-						const tmpElapsed = Date.now() - tmpStartTime;
-						this.fable.log.info(`Delete sync time budget exhausted for ${this.EntitySchema.TableName} after ${tmpElapsed}ms (${tmpProcessed} of ${tmpDeletedCount} deleted records processed).`);
-						return fCallback();
+						return fFinish('time budget exhausted');
 					}
-
-					// All pages processed
 					if (tmpOffset >= tmpDeleteCap)
 					{
-						const tmpElapsed = Date.now() - tmpStartTime;
-						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed in ${tmpElapsed}ms).`);
-						return fCallback();
+						return fFinish('complete');
 					}
 
-					const tmpURL = this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${tmpOffset}/${this.PageSize}`);
+					const tmpURL = this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~DESC~DESC/${tmpOffset}/${this.PageSize}`);
 					tmpOffset += this.PageSize;
 
 					this.fable.MeadowCloneRestClient.getJSON(tmpURL,
-						(pDownloadError, pResponse, pBody) =>
+						(pDownloadError, pResponse, pPageBody) =>
 						{
-							if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
+							if (pDownloadError || !pPageBody || !Array.isArray(pPageBody) || pPageBody.length < 1)
 							{
-								// Empty page or error — we've reached the end
-								const tmpElapsed = Date.now() - tmpStartTime;
-								this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed in ${tmpElapsed}ms).`);
-								return fCallback();
+								return fFinish('complete');
 							}
 
-							this.fable.Utility.eachLimit(pBody, 5,
+							this.fable.Utility.eachLimit(pPageBody, 5,
 								(pEntityRecord, fRecordComplete) =>
 								{
-									const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
-									if (!tmpRecordID || tmpRecordID < 1)
-									{
-										return setImmediate(fRecordComplete);
-									}
-
-									const tmpQuery = this.Meadow.query;
-									tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
-									tmpQuery.setDisableDeleteTracking(true);
-
-									this.Meadow.doRead(tmpQuery,
-										(pReadError, pQuery, pRecord) =>
-										{
-											if (pReadError || !pRecord)
-											{
-												const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
-
-												const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
-												tmpCreateQuery.setDisableAutoIdentity(true);
-												tmpCreateQuery.setDisableAutoDateStamp(true);
-												tmpCreateQuery.setDisableAutoUserStamp(true);
-												tmpCreateQuery.setDisableDeleteTracking(true);
-												tmpCreateQuery.AllowIdentityInsert = true;
-
-												this.Meadow.doCreate(tmpCreateQuery,
-													(pCreateError) =>
-													{
-														if (pCreateError)
-														{
-															this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
-														}
-														tmpProcessed++;
-														return setImmediate(fRecordComplete);
-													});
-												return;
-											}
-
-											if (pRecord.Deleted == 1)
-											{
-												tmpProcessed++;
-												return setImmediate(fRecordComplete);
-											}
-
-											const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
-
-											const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
-											tmpUpdateQuery.setDisableAutoIdentity(true);
-											tmpUpdateQuery.setDisableAutoDateStamp(true);
-											tmpUpdateQuery.setDisableAutoUserStamp(true);
-											tmpUpdateQuery.setDisableDeleteTracking(true);
-
-											this.Meadow.doUpdate(tmpUpdateQuery,
-												(pUpdateError) =>
-												{
-													if (pUpdateError)
-													{
-														this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
-													}
-													tmpProcessed++;
-													return setImmediate(fRecordComplete);
-												});
-										});
+									tmpCounters.seen++;
+									this._reconcileDeletedRecordByID(pEntityRecord, tmpCounters, fRecordComplete);
 								},
 								(pRecordSyncError) =>
 								{
-									// Page complete — fetch next page
+									// Page complete — heartbeat, then fetch next page.
+									const tmpElapsed = Date.now() - tmpStartTime;
+									this.fable.log.info(`Delete sync ${this.EntitySchema.TableName}: examined ${tmpCounters.seen}/${tmpDeletedCount} — marked ${tmpCounters.marked}, already ${tmpCounters.already}, not-in-clone ${tmpCounters.notInClone}, errors ${tmpCounters.errors} (${tmpElapsed}ms).`);
 									return setImmediate(fFetchDeletedPage);
 								});
 						});
@@ -294,6 +280,255 @@ class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngo
 			});
 	}
 
+	// Reconcile one server-deleted record into the clone, matched by IDENTITY
+	// ONLY (never GUID — see the SAFETY note on syncDeletedRecords).  Increments
+	// exactly one counter on pCounters: marked / already / notInClone / errors.
+	// Shared by both the time-budgeted path and the resumable-cursor path.
+	_reconcileDeletedRecordByID(pEntityRecord, pCounters, fDone)
+	{
+		const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
+		if (tmpRecordID === undefined || tmpRecordID === null || tmpRecordID < 1)
+		{
+			pCounters.notInClone++;
+			return setImmediate(fDone);
+		}
+
+		// Read by the authoritative, unique identity column.  Delete tracking is
+		// disabled on the read so an already-deleted local row is still found
+		// (and skipped) rather than re-attempted.
+		const tmpQuery = this.Meadow.query;
+		tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
+		tmpQuery.setDisableDeleteTracking(true);
+
+		this.Meadow.doRead(tmpQuery,
+			(pReadError, pReadQuery, pLocalRecord) =>
+			{
+				if (pReadError)
+				{
+					pCounters.errors++;
+					this.log.error(`Delete sync read error for ${this.EntitySchema.TableName} ${this.DefaultIdentifier}=${tmpRecordID}: ${pReadError}`);
+					return setImmediate(fDone);
+				}
+
+				// Deleted id not in the clone — deleted before it was ever synced
+				// here.  Do NOT create (collides on duplicate GUIDs); leave it for
+				// the backfill/de-dup cleanup.
+				if (!pLocalRecord)
+				{
+					pCounters.notInClone++;
+					return setImmediate(fDone);
+				}
+
+				// Already reconciled — cheap skip.
+				if (pLocalRecord.Deleted == 1)
+				{
+					pCounters.already++;
+					return setImmediate(fDone);
+				}
+
+				// Flag THIS row — selected by its authoritative, unique id, so we
+				// can never touch a different record that shares the GUID.  doDelete
+				// is the canonical soft-delete (UPDATE ... SET Deleted=1,
+				// DeleteDate=NOW(), DeletingIDUser=... WHERE id=? AND Deleted=0):
+				// idempotent, and — unlike doUpdate — it neither strips the
+				// delete-tracking columns nor trips the delete-tracking-filtered
+				// post-update verify read.  DeleteDate is the local detection time
+				// (meadow has no path to set the source's value); fine for the clone.
+				const tmpDeleteQuery = this.Meadow.query;
+				tmpDeleteQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
+
+				this.Meadow.doDelete(tmpDeleteQuery,
+					(pDeleteError) =>
+					{
+						if (pDeleteError)
+						{
+							pCounters.errors++;
+							this.log.error(`Error marking record deleted ${this.EntitySchema.TableName} ${this.DefaultIdentifier}=${tmpRecordID}: ${pDeleteError}`);
+						}
+						else
+						{
+							pCounters.marked++;
+						}
+						return setImmediate(fDone);
+					});
+			});
+	}
+
+	// Resumable delete reconciliation (opt-in via DeleteCursorStatePath).
+	//
+	// Persists two id marks per table in a small JSON file so a run resumes
+	// rather than re-walking from the newest record (which, with heavy rows and a
+	// time budget, never reaches the older backlog):
+	//   - HeadID: highest deleted id already covered from the top.
+	//   - TailID: resume point of the downward catch-up sweep.
+	// Each run does a HEAD pass (id > HeadID — new deletions since last run, cheap)
+	// then a TAIL pass (id < TailID — drain older backlog within the remaining
+	// budget).  When the tail reaches the bottom, the backlog is drained and only
+	// the head pass runs thereafter.  A periodic re-sweep (DeleteResweepIntervalHours)
+	// resets the tail to catch deletions that landed in already-swept id ranges.
+	// All keyset paging (id < cursor) — no growing OFFSET scan.
+	//
+	// Safety/behavior is identical to the non-cursor path (same id-only
+	// _reconcileDeletedRecordByID); only WHERE paging starts differs.  Missing or
+	// unreadable state degrades to a full sweep.
+	_syncDeletedRecordsWithCursor(fCallback)
+	{
+		const tmpTable = this.EntitySchema.TableName;
+		const tmpStore = new libMeadowDeleteCursorStore(this.DeleteCursorStatePath, this.fable.log);
+		const tmpState = tmpStore.get(tmpTable) || { HeadID: 0, TailID: null, CaughtUp: false, LastSweepEpoch: 0 };
+
+		// Re-sweep: once caught up, periodically reset the tail to re-drain from
+		// the top so deletions that landed in already-swept ranges are caught.
+		const tmpNow = Date.now();
+		if (tmpState.CaughtUp && this.DeleteResweepIntervalHours > 0
+			&& (tmpNow - (tmpState.LastSweepEpoch || 0)) >= (this.DeleteResweepIntervalHours * 3600000))
+		{
+			this.fable.log.info(`Delete cursor for ${tmpTable}: re-sweep interval elapsed; resetting tail to re-drain from the top.`);
+			tmpState.TailID = null;
+			tmpState.CaughtUp = false;
+		}
+
+		this.fable.log.info(`Delete cursor for ${tmpTable}: headID=${tmpState.HeadID}, tailID=${tmpState.TailID === null ? 'top' : tmpState.TailID}, caughtUp=${tmpState.CaughtUp} (matching on ${this.DefaultIdentifier}, ${this.BackSyncTimeLimit}ms budget)...`);
+
+		const tmpStartTime = Date.now();
+		const tmpCounters = { seen: 0, marked: 0, already: 0, notInClone: 0, errors: 0 };
+		let tmpNewHeadID = tmpState.HeadID;
+
+		const fSaveAndFinish = (pReason) =>
+		{
+			tmpState.HeadID = tmpNewHeadID;
+			tmpStore.set(tmpTable, tmpState);
+			const tmpElapsed = Date.now() - tmpStartTime;
+			if (this.syncResults)
+			{
+				this.syncResults.Deleted = tmpCounters.marked;
+			}
+			this.fable.log.info(`Delete cursor ${pReason} for ${tmpTable} after ${tmpElapsed}ms: marked ${tmpCounters.marked}, already ${tmpCounters.already}, not-in-clone ${tmpCounters.notInClone}, errors ${tmpCounters.errors} (examined ${tmpCounters.seen}); headID=${tmpState.HeadID}, tailID=${tmpState.TailID === null ? 'top' : tmpState.TailID}, caughtUp=${tmpState.CaughtUp}.`);
+			return fCallback();
+		};
+
+		// HEAD pass — only once a head has been established (not the first run).
+		const fRunHeadPass = (fHeadDone) =>
+		{
+			if (!(tmpState.HeadID > 0))
+			{
+				return fHeadDone();
+			}
+			this._fetchDeletedKeysetPass({ floorID: tmpState.HeadID, ceilID: null, startTime: tmpStartTime, counters: tmpCounters, label: 'head' },
+				(pResult) =>
+				{
+					if (pResult.maxID !== null && pResult.maxID > tmpNewHeadID) { tmpNewHeadID = pResult.maxID; }
+					return fHeadDone();
+				});
+		};
+
+		// TAIL pass — drain downward from TailID (null = from the very top, which
+		// is the first run and also establishes the head).
+		const fRunTailPass = (fTailDone) =>
+		{
+			if (tmpState.CaughtUp)
+			{
+				return fTailDone();
+			}
+			this._fetchDeletedKeysetPass({ floorID: null, ceilID: tmpState.TailID, startTime: tmpStartTime, counters: tmpCounters, label: 'tail' },
+				(pResult) =>
+				{
+					if (pResult.maxID !== null && pResult.maxID > tmpNewHeadID) { tmpNewHeadID = pResult.maxID; }
+					if (pResult.minID !== null) { tmpState.TailID = pResult.minID; }
+					if (pResult.reachedEnd)
+					{
+						tmpState.CaughtUp = true;
+						tmpState.LastSweepEpoch = Date.now();
+					}
+					return fTailDone();
+				});
+		};
+
+		fRunHeadPass(() =>
+		{
+			fRunTailPass(() =>
+			{
+				return fSaveAndFinish(tmpState.CaughtUp ? 'caught up (steady state)' : 'progressed');
+			});
+		});
+	}
+
+	// Keyset-paged pass over the deleted set: Deleted=1 [AND id > floorID]
+	// [AND id < ceilID], ordered id DESC, advancing the ceiling to each page's
+	// lowest id.  Shares the run's time budget (BackSyncTimeLimit) and the global
+	// MaxRecordsPerEntity cap.  Calls fComplete({ maxID, minID, reachedEnd }):
+	// maxID = highest id seen (first row, establishes head on the first run),
+	// minID = lowest id seen (next resume point), reachedEnd = the deleted set was
+	// exhausted within this pass.
+	_fetchDeletedKeysetPass(pOptions, fComplete)
+	{
+		const tmpFloorID = (pOptions.floorID === undefined) ? null : pOptions.floorID;
+		let tmpCeilID = (pOptions.ceilID === undefined) ? null : pOptions.ceilID;
+		const tmpCounters = pOptions.counters;
+		let tmpMaxID = null;
+		let tmpMinID = null;
+
+		const fFetch = () =>
+		{
+			if (Date.now() - pOptions.startTime >= this.BackSyncTimeLimit)
+			{
+				return fComplete({ maxID: tmpMaxID, minID: tmpMinID, reachedEnd: false });
+			}
+			if (this.MaxRecordsPerEntity > 0 && tmpCounters.seen >= this.MaxRecordsPerEntity)
+			{
+				return fComplete({ maxID: tmpMaxID, minID: tmpMinID, reachedEnd: false });
+			}
+
+			let tmpFilter = 'FBV~Deleted~EQ~1';
+			if (tmpFloorID !== null) { tmpFilter += `~FBV~${this.DefaultIdentifier}~GT~${tmpFloorID}`; }
+			if (tmpCeilID !== null) { tmpFilter += `~FBV~${this.DefaultIdentifier}~LT~${tmpCeilID}`; }
+			tmpFilter += `~FSF~${this.DefaultIdentifier}~DESC~DESC`;
+			const tmpURL = this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/${tmpFilter}/0/${this.PageSize}`);
+
+			this.fable.MeadowCloneRestClient.getJSON(tmpURL,
+				(pError, pResponse, pPageBody) =>
+				{
+					if (pError)
+					{
+						this.fable.log.warn(`Delete cursor ${this.EntitySchema.TableName} [${pOptions.label}]: page fetch error (${pError}); pausing pass.`);
+						return fComplete({ maxID: tmpMaxID, minID: tmpMinID, reachedEnd: false });
+					}
+					if (!Array.isArray(pPageBody) || pPageBody.length < 1)
+					{
+						// Empty page = exhausted this id range.
+						return fComplete({ maxID: tmpMaxID, minID: tmpMinID, reachedEnd: true });
+					}
+
+					if (tmpMaxID === null)
+					{
+						tmpMaxID = pPageBody[0][this.DefaultIdentifier]; // DESC → first row is the highest id
+					}
+
+					this.fable.Utility.eachLimit(pPageBody, 5,
+						(pEntityRecord, fRecordComplete) =>
+						{
+							tmpCounters.seen++;
+							this._reconcileDeletedRecordByID(pEntityRecord, tmpCounters, fRecordComplete);
+						},
+						(pRecordSyncError) =>
+						{
+							const tmpPageMinID = pPageBody[pPageBody.length - 1][this.DefaultIdentifier];
+							tmpMinID = tmpPageMinID;
+							tmpCeilID = tmpPageMinID; // keyset: next page is strictly below this page's lowest id
+							const tmpElapsed = Date.now() - pOptions.startTime;
+							this.fable.log.info(`Delete cursor ${this.EntitySchema.TableName} [${pOptions.label}]: examined ${tmpCounters.seen} — marked ${tmpCounters.marked}, already ${tmpCounters.already}, not-in-clone ${tmpCounters.notInClone}, errors ${tmpCounters.errors}; at id ${tmpPageMinID} (${tmpElapsed}ms).`);
+							if (pPageBody.length < this.PageSize)
+							{
+								return fComplete({ maxID: tmpMaxID, minID: tmpMinID, reachedEnd: true });
+							}
+							return setImmediate(fFetch);
+						});
+				});
+		};
+
+		fFetch();
+	}
+
 	_syncInternal(fCallback)
 	{
 		this.operation.createTimeStamp('EntityOngoingEventualConsistencySync');

package/source/services/clone/Meadow-Service-Sync.js

@@ -184,6 +184,8 @@ class MeadowSync extends libFableServiceProviderBase
 						UseAdvancedIDPagination: this.UseAdvancedIDPagination,
 						BackSyncTimeLimit: this.BackSyncTimeLimit,
 						TrueUpPageSize: this.TrueUpPageSize,
+						DeleteCursorStatePath: this.options.DeleteCursorStatePath,
+						DeleteResweepIntervalHours: this.options.DeleteResweepIntervalHours,
 					};
 
 					// Apply per-entity option overrides if configured

package/test/Meadow-Integration-BisectionSync_test.js

@@ -428,18 +428,28 @@ function setupSQLiteProvider(pFable, fCallback)
 
 function seedLocalBooks(pFable, pBooks)
 {
-	const tmpInsert = pFable.MeadowSQLiteProvider.db.prepare(`
+	// node:sqlite's DatabaseSync has no `.transaction(fn)` helper (that was
+	// a better-sqlite3 idiom).  Bracket the bulk insert manually so seeding
+	// many thousands of rows doesn't pay per-row commit overhead.
+	const tmpDB = pFable.MeadowSQLiteProvider.db;
+	const tmpInsert = tmpDB.prepare(`
 		INSERT OR REPLACE INTO Book (IDBook, GUIDBook, CreateDate, CreatingIDUser, UpdateDate, UpdatingIDUser, Deleted, DeleteDate, DeletingIDUser, Title, Type, Genre, PublicationYear)
 		VALUES (@IDBook, @GUIDBook, @CreateDate, @CreatingIDUser, @UpdateDate, @UpdatingIDUser, @Deleted, @DeleteDate, @DeletingIDUser, @Title, @Type, @Genre, @PublicationYear)
 	`);
-	const tmpInsertMany = pFable.MeadowSQLiteProvider.db.transaction((pRecords) =>
+	tmpDB.exec('BEGIN');
+	try
 	{
-		for (const tmpRecord of pRecords)
+		for (const tmpRecord of pBooks)
 		{
 			tmpInsert.run(tmpRecord);
 		}
-	});
-	tmpInsertMany(pBooks);
+		tmpDB.exec('COMMIT');
+	}
+	catch (pError)
+	{
+		tmpDB.exec('ROLLBACK');
+		throw pError;
+	}
 }
 
 function getLocalBookCount(pFable)