meadow-integration 1.0.28 → 1.0.30

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "meadow-integration",
-    "version": "1.0.28",
+    "version": "1.0.30",
     "description": "Meadow Data Integration",
     "bin": {
         "mdwint": "source/cli/Meadow-Integration-CLI-Run.js"
@@ -40,7 +40,7 @@
         ]
     },
     "dependencies": {
-        "fable": "^3.1.67",
+        "fable": "^3.1.70",
         "fable-serviceproviderbase": "^3.0.19",
         "fast-xml-parser": "^4.4.1",
         "meadow": "^2.0.33",

package/source/services/clone/Meadow-Service-RestClient.js

@@ -1,6 +1,4 @@
 const libFableServiceProviderBase = require('fable-serviceproviderbase');
-const Http = require('http');
-const Https = require('https');
 
 const defaultRestClientOptions = (
 	{
@@ -41,30 +39,22 @@ class MeadowCloneRestClient extends libFableServiceProviderBase
 			this._SessionToken = this.options.SessionToken;
 		}
 
-		this.restClient = this.fable.serviceManager.instantiateServiceProvider('RestClient', {}, 'MeadowCloneRestClient-RestClient');
-		this.cache = {};
-
-		this.requestTimeout = this.options.RequestTimeout;
-		this.maxRequestTimeout = this.options.MaxRequestTimeout;
+		// Fable settings override instance options for timeouts
+		this.requestTimeout = this.fable.settings.RestClientRequestTimeout || this.options.RequestTimeout;
+		this.maxRequestTimeout = this.fable.settings.RestClientMaxRequestTimeout || this.options.MaxRequestTimeout;
 
-		// Use the longer of the two timeouts for the agent's socket timeout
-		// so that MAX queries don't get killed at the socket level.
-		const agentOptions = { keepAlive: true, timeout: Math.max(this.requestTimeout, this.maxRequestTimeout) };
-
-		if (this.serverURL && this.serverURL.startsWith('http:'))
-		{
-			this.agent = new Http.Agent(agentOptions);
-		}
-		else
-		{
-			this.agent = new Https.Agent(agentOptions);
-		}
-
-		this.restClient.prepareRequestOptions = (pOptions) =>
-		{
-			pOptions.agent = this.agent;
-			return pOptions;
-		};
+		// When keep-alive is enabled via fable settings (RestClientKeepAlive), use a
+		// socket timeout based on the longer of the two request timeouts so that
+		// slow MAX queries don't get killed at the socket level.
+		let tmpRestClientOptions = (
+			{
+				KeepAliveAgentOptions:
+					{
+						timeout: Math.max(this.requestTimeout, this.maxRequestTimeout)
+					}
+			});
+		this.restClient = this.fable.serviceManager.instantiateServiceProvider('RestClient', tmpRestClientOptions, 'MeadowCloneRestClient-RestClient');
+		this.cache = {};
 	}
 
 	prepareRequestOptions(pOptions)

package/source/services/clone/Meadow-Service-Sync-Entity-OngoingEventualConsistency.js

@@ -174,119 +174,123 @@ class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngo
 				let tmpDeleteCap = (this.MaxRecordsPerEntity > 0)
 					? Math.min(tmpDeletedCount, this.MaxRecordsPerEntity)
 					: tmpDeletedCount;
-				const tmpDeleteURLPartials = [];
-				for (let i = 0; i < tmpDeleteCap; i += this.PageSize)
-				{
-					tmpDeleteURLPartials.push(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`));
-				}
 
 				const tmpStartTime = Date.now();
 				let tmpProcessed = 0;
-				let tmpTimeBudgetExhausted = false;
+				let tmpOffset = 0;
+
+				// Fetch deleted record pages one at a time using a recursive
+				// fetcher instead of pre-generating all URL partials.  With
+				// millions of deleted records and a short time budget, the old
+				// eachLimit approach generated hundreds of thousands of partials
+				// and then had to skip them all when the budget expired.
+				const fFetchDeletedPage = () =>
+				{
+					// Time budget check — stop immediately
+					if (Date.now() - tmpStartTime >= this.BackSyncTimeLimit)
+					{
+						const tmpElapsed = Date.now() - tmpStartTime;
+						this.fable.log.info(`Delete sync time budget exhausted for ${this.EntitySchema.TableName} after ${tmpElapsed}ms (${tmpProcessed} of ${tmpDeletedCount} deleted records processed).`);
+						return fCallback();
+					}
 
-				this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
-					(pURLPartial, fPageComplete) =>
+					// All pages processed
+					if (tmpOffset >= tmpDeleteCap)
 					{
-						// Check time budget before each page
-						if (Date.now() - tmpStartTime >= this.BackSyncTimeLimit)
-						{
-							tmpTimeBudgetExhausted = true;
-							return fPageComplete();
-						}
+						const tmpElapsed = Date.now() - tmpStartTime;
+						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed in ${tmpElapsed}ms).`);
+						return fCallback();
+					}
+
+					const tmpURL = this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${tmpOffset}/${this.PageSize}`);
+					tmpOffset += this.PageSize;
 
-						this.fable.MeadowCloneRestClient.getJSON(pURLPartial,
-							(pDownloadError, pResponse, pBody) =>
+					this.fable.MeadowCloneRestClient.getJSON(tmpURL,
+						(pDownloadError, pResponse, pBody) =>
+						{
+							if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
 							{
-								if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
-								{
-									return fPageComplete();
-								}
+								// Empty page or error — we've reached the end
+								const tmpElapsed = Date.now() - tmpStartTime;
+								this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed in ${tmpElapsed}ms).`);
+								return fCallback();
+							}
 
-								this.fable.Utility.eachLimit(pBody, 5,
-									(pEntityRecord, fRecordComplete) =>
+							this.fable.Utility.eachLimit(pBody, 5,
+								(pEntityRecord, fRecordComplete) =>
+								{
+									const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
+									if (!tmpRecordID || tmpRecordID < 1)
 									{
-										const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
-										if (!tmpRecordID || tmpRecordID < 1)
-										{
-											return setImmediate(fRecordComplete);
-										}
+										return setImmediate(fRecordComplete);
+									}
 
-										const tmpQuery = this.Meadow.query;
-										tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
-										tmpQuery.setDisableDeleteTracking(true);
+									const tmpQuery = this.Meadow.query;
+									tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
+									tmpQuery.setDisableDeleteTracking(true);
 
-										this.Meadow.doRead(tmpQuery,
-											(pReadError, pQuery, pRecord) =>
+									this.Meadow.doRead(tmpQuery,
+										(pReadError, pQuery, pRecord) =>
+										{
+											if (pReadError || !pRecord)
 											{
-												if (pReadError || !pRecord)
-												{
-													const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
-
-													const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
-													tmpCreateQuery.setDisableAutoIdentity(true);
-													tmpCreateQuery.setDisableAutoDateStamp(true);
-													tmpCreateQuery.setDisableAutoUserStamp(true);
-													tmpCreateQuery.setDisableDeleteTracking(true);
-													tmpCreateQuery.AllowIdentityInsert = true;
-
-													this.Meadow.doCreate(tmpCreateQuery,
-														(pCreateError) =>
-														{
-															if (pCreateError)
-															{
-																this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
-															}
-															tmpProcessed++;
-															return setImmediate(fRecordComplete);
-														});
-													return;
-												}
-
-												if (pRecord.Deleted == 1)
-												{
-													tmpProcessed++;
-													return setImmediate(fRecordComplete);
-												}
-
 												const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
 
-												const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
-												tmpUpdateQuery.setDisableAutoIdentity(true);
-												tmpUpdateQuery.setDisableAutoDateStamp(true);
-												tmpUpdateQuery.setDisableAutoUserStamp(true);
-												tmpUpdateQuery.setDisableDeleteTracking(true);
+												const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+												tmpCreateQuery.setDisableAutoIdentity(true);
+												tmpCreateQuery.setDisableAutoDateStamp(true);
+												tmpCreateQuery.setDisableAutoUserStamp(true);
+												tmpCreateQuery.setDisableDeleteTracking(true);
+												tmpCreateQuery.AllowIdentityInsert = true;
 
-												this.Meadow.doUpdate(tmpUpdateQuery,
-													(pUpdateError) =>
+												this.Meadow.doCreate(tmpCreateQuery,
+													(pCreateError) =>
 													{
-														if (pUpdateError)
+														if (pCreateError)
 														{
-															this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+															this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
 														}
 														tmpProcessed++;
 														return setImmediate(fRecordComplete);
 													});
-											});
-									},
-									(pRecordSyncError) =>
-									{
-										return fPageComplete();
-									});
-							});
-					},
-					(pDeleteSyncError) =>
-					{
-						const tmpElapsed = Date.now() - tmpStartTime;
-						if (tmpTimeBudgetExhausted)
-						{
-							this.fable.log.info(`Delete sync time budget exhausted for ${this.EntitySchema.TableName} after ${tmpElapsed}ms (${tmpProcessed} of ${tmpDeletedCount} deleted records processed).`);
-						}
-						else
-						{
-							this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed in ${tmpElapsed}ms).`);
-						}
-						return fCallback();
-					});
+												return;
+											}
+
+											if (pRecord.Deleted == 1)
+											{
+												tmpProcessed++;
+												return setImmediate(fRecordComplete);
+											}
+
+											const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+											const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+											tmpUpdateQuery.setDisableAutoIdentity(true);
+											tmpUpdateQuery.setDisableAutoDateStamp(true);
+											tmpUpdateQuery.setDisableAutoUserStamp(true);
+											tmpUpdateQuery.setDisableDeleteTracking(true);
+
+											this.Meadow.doUpdate(tmpUpdateQuery,
+												(pUpdateError) =>
+												{
+													if (pUpdateError)
+													{
+														this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+													}
+													tmpProcessed++;
+													return setImmediate(fRecordComplete);
+												});
+										});
+								},
+								(pRecordSyncError) =>
+								{
+									// Page complete — fetch next page
+									return setImmediate(fFetchDeletedPage);
+								});
+						});
+				};
+
+				fFetchDeletedPage();
 			});
 	}