meadow-integration 1.0.29 → 1.0.31

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "meadow-integration",
-    "version": "1.0.29",
+    "version": "1.0.31",
     "description": "Meadow Data Integration",
     "bin": {
         "mdwint": "source/cli/Meadow-Integration-CLI-Run.js"

package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js

@@ -688,105 +688,115 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 
 				this.fable.log.info(`Found ${tmpDeletedCount} deleted records on server for ${this.EntitySchema.TableName}; syncing deletions...`);
 
-				// Generate paginated URLs for deleted records
 				let tmpDeleteCap = (this.MaxRecordsPerEntity > 0)
 					? Math.min(tmpDeletedCount, this.MaxRecordsPerEntity)
 					: tmpDeletedCount;
-				const tmpDeleteURLPartials = [];
-				for (let i = 0; i < tmpDeleteCap; i += this.PageSize)
-				{
-					tmpDeleteURLPartials.push(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`));
-				}
 
-				this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
-					(pURLPartial, fPageComplete) =>
+				let tmpOffset = 0;
+				let tmpProcessed = 0;
+
+				// Fetch deleted record pages one at a time rather than
+				// pre-generating all URL partials.  With millions of deleted
+				// records this avoids allocating hundreds of thousands of
+				// URL strings and prevents stack overflow in the iterator.
+				const fFetchDeletedPage = () =>
+				{
+					if (tmpOffset >= tmpDeleteCap)
 					{
-						this.fable.MeadowCloneRestClient.getJSON(pURLPartial,
-							(pDownloadError, pResponse, pBody) =>
+						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed).`);
+						return fCallback();
+					}
+
+					const tmpURL = this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${tmpOffset}/${this.PageSize}`);
+					tmpOffset += this.PageSize;
+
+					this.fable.MeadowCloneRestClient.getJSON(tmpURL,
+						(pDownloadError, pResponse, pBody) =>
+						{
+							if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
 							{
-								if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
-								{
-									return fPageComplete();
-								}
+								this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed).`);
+								return fCallback();
+							}
 
-								this.fable.Utility.eachLimit(pBody, 5,
-									(pEntityRecord, fRecordComplete) =>
+							this.fable.Utility.eachLimit(pBody, 5,
+								(pEntityRecord, fRecordComplete) =>
+								{
+									const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
+									if (!tmpRecordID || tmpRecordID < 1)
 									{
-										const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
-										if (!tmpRecordID || tmpRecordID < 1)
-										{
-											return setImmediate(fRecordComplete);
-										}
+										return setImmediate(fRecordComplete);
+									}
 
-										// Read local record with delete tracking disabled so we can see all records
-										const tmpQuery = this.Meadow.query;
-										tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
-										tmpQuery.setDisableDeleteTracking(true);
+									// Read local record with delete tracking disabled so we can see all records
+									const tmpQuery = this.Meadow.query;
+									tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
+									tmpQuery.setDisableDeleteTracking(true);
 
-										this.Meadow.doRead(tmpQuery,
-											(pReadError, pQuery, pRecord) =>
+									this.Meadow.doRead(tmpQuery,
+										(pReadError, pQuery, pRecord) =>
+										{
+											if (pReadError || !pRecord)
 											{
-												if (pReadError || !pRecord)
-												{
-													// Record doesn't exist locally -- create it as deleted
-													const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
-
-													const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
-													tmpCreateQuery.setDisableAutoIdentity(true);
-													tmpCreateQuery.setDisableAutoDateStamp(true);
-													tmpCreateQuery.setDisableAutoUserStamp(true);
-													tmpCreateQuery.setDisableDeleteTracking(true);
-													tmpCreateQuery.AllowIdentityInsert = true;
-
-													this.Meadow.doCreate(tmpCreateQuery,
-														(pCreateError) =>
-														{
-															if (pCreateError)
-															{
-																this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
-															}
-															return setImmediate(fRecordComplete);
-														});
-													return;
-												}
-
-												if (pRecord.Deleted == 1)
-												{
-													// Already marked deleted locally
-													return setImmediate(fRecordComplete);
-												}
-
-												// Record exists locally but is not deleted -- update it
+												// Record doesn't exist locally -- create it as deleted
 												const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
 
-												const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
-												tmpUpdateQuery.setDisableAutoIdentity(true);
-												tmpUpdateQuery.setDisableAutoDateStamp(true);
-												tmpUpdateQuery.setDisableAutoUserStamp(true);
-												tmpUpdateQuery.setDisableDeleteTracking(true);
+												const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+												tmpCreateQuery.setDisableAutoIdentity(true);
+												tmpCreateQuery.setDisableAutoDateStamp(true);
+												tmpCreateQuery.setDisableAutoUserStamp(true);
+												tmpCreateQuery.setDisableDeleteTracking(true);
+												tmpCreateQuery.AllowIdentityInsert = true;
 
-												this.Meadow.doUpdate(tmpUpdateQuery,
-													(pUpdateError) =>
+												this.Meadow.doCreate(tmpCreateQuery,
+													(pCreateError) =>
 													{
-														if (pUpdateError)
+														if (pCreateError)
 														{
-															this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+															this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
 														}
+														tmpProcessed++;
 														return setImmediate(fRecordComplete);
 													});
-											});
-									},
-									(pRecordSyncError) =>
-									{
-										return fPageComplete();
-									});
-							});
-					},
-					(pDeleteSyncError) =>
-					{
-						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpDeletedCount} deleted records processed).`);
-						return fCallback();
-					});
+												return;
+											}
+
+											if (pRecord.Deleted == 1)
+											{
+												// Already marked deleted locally
+												tmpProcessed++;
+												return setImmediate(fRecordComplete);
+											}
+
+											// Record exists locally but is not deleted -- update it
+											const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+											const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+											tmpUpdateQuery.setDisableAutoIdentity(true);
+											tmpUpdateQuery.setDisableAutoDateStamp(true);
+											tmpUpdateQuery.setDisableAutoUserStamp(true);
+											tmpUpdateQuery.setDisableDeleteTracking(true);
+
+											this.Meadow.doUpdate(tmpUpdateQuery,
+												(pUpdateError) =>
+												{
+													if (pUpdateError)
+													{
+														this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+													}
+													tmpProcessed++;
+													return setImmediate(fRecordComplete);
+												});
+										});
+								},
+								(pRecordSyncError) =>
+								{
+									return setImmediate(fFetchDeletedPage);
+								});
+						});
+				};
+
+				fFetchDeletedPage();
 			});
 	}
 

package/source/services/clone/Meadow-Service-Sync-Entity-OngoingEventualConsistency.js

@@ -174,119 +174,123 @@ class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngo
 				let tmpDeleteCap = (this.MaxRecordsPerEntity > 0)
 					? Math.min(tmpDeletedCount, this.MaxRecordsPerEntity)
 					: tmpDeletedCount;
-				const tmpDeleteURLPartials = [];
-				for (let i = 0; i < tmpDeleteCap; i += this.PageSize)
-				{
-					tmpDeleteURLPartials.push(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`));
-				}
 
 				const tmpStartTime = Date.now();
 				let tmpProcessed = 0;
-				let tmpTimeBudgetExhausted = false;
+				let tmpOffset = 0;
+
+				// Fetch deleted record pages one at a time using a recursive
+				// fetcher instead of pre-generating all URL partials.  With
+				// millions of deleted records and a short time budget, the old
+				// eachLimit approach generated hundreds of thousands of partials
+				// and then had to skip them all when the budget expired.
+				const fFetchDeletedPage = () =>
+				{
+					// Time budget check — stop immediately
+					if (Date.now() - tmpStartTime >= this.BackSyncTimeLimit)
+					{
+						const tmpElapsed = Date.now() - tmpStartTime;
+						this.fable.log.info(`Delete sync time budget exhausted for ${this.EntitySchema.TableName} after ${tmpElapsed}ms (${tmpProcessed} of ${tmpDeletedCount} deleted records processed).`);
+						return fCallback();
+					}
 
-				this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
-					(pURLPartial, fPageComplete) =>
+					// All pages processed
+					if (tmpOffset >= tmpDeleteCap)
 					{
-						// Check time budget before each page
-						if (Date.now() - tmpStartTime >= this.BackSyncTimeLimit)
-						{
-							tmpTimeBudgetExhausted = true;
-							return fPageComplete();
-						}
+						const tmpElapsed = Date.now() - tmpStartTime;
+						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed in ${tmpElapsed}ms).`);
+						return fCallback();
+					}
+
+					const tmpURL = this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${tmpOffset}/${this.PageSize}`);
+					tmpOffset += this.PageSize;
 
-						this.fable.MeadowCloneRestClient.getJSON(pURLPartial,
-							(pDownloadError, pResponse, pBody) =>
+					this.fable.MeadowCloneRestClient.getJSON(tmpURL,
+						(pDownloadError, pResponse, pBody) =>
+						{
+							if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
 							{
-								if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
-								{
-									return fPageComplete();
-								}
+								// Empty page or error — we've reached the end
+								const tmpElapsed = Date.now() - tmpStartTime;
+								this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed in ${tmpElapsed}ms).`);
+								return fCallback();
+							}
 
-								this.fable.Utility.eachLimit(pBody, 5,
-									(pEntityRecord, fRecordComplete) =>
+							this.fable.Utility.eachLimit(pBody, 5,
+								(pEntityRecord, fRecordComplete) =>
+								{
+									const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
+									if (!tmpRecordID || tmpRecordID < 1)
 									{
-										const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
-										if (!tmpRecordID || tmpRecordID < 1)
-										{
-											return setImmediate(fRecordComplete);
-										}
+										return setImmediate(fRecordComplete);
+									}
 
-										const tmpQuery = this.Meadow.query;
-										tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
-										tmpQuery.setDisableDeleteTracking(true);
+									const tmpQuery = this.Meadow.query;
+									tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
+									tmpQuery.setDisableDeleteTracking(true);
 
-										this.Meadow.doRead(tmpQuery,
-											(pReadError, pQuery, pRecord) =>
+									this.Meadow.doRead(tmpQuery,
+										(pReadError, pQuery, pRecord) =>
+										{
+											if (pReadError || !pRecord)
 											{
-												if (pReadError || !pRecord)
-												{
-													const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
-
-													const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
-													tmpCreateQuery.setDisableAutoIdentity(true);
-													tmpCreateQuery.setDisableAutoDateStamp(true);
-													tmpCreateQuery.setDisableAutoUserStamp(true);
-													tmpCreateQuery.setDisableDeleteTracking(true);
-													tmpCreateQuery.AllowIdentityInsert = true;
-
-													this.Meadow.doCreate(tmpCreateQuery,
-														(pCreateError) =>
-														{
-															if (pCreateError)
-															{
-																this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
-															}
-															tmpProcessed++;
-															return setImmediate(fRecordComplete);
-														});
-													return;
-												}
-
-												if (pRecord.Deleted == 1)
-												{
-													tmpProcessed++;
-													return setImmediate(fRecordComplete);
-												}
-
 												const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
 
-												const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
-												tmpUpdateQuery.setDisableAutoIdentity(true);
-												tmpUpdateQuery.setDisableAutoDateStamp(true);
-												tmpUpdateQuery.setDisableAutoUserStamp(true);
-												tmpUpdateQuery.setDisableDeleteTracking(true);
+												const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+												tmpCreateQuery.setDisableAutoIdentity(true);
+												tmpCreateQuery.setDisableAutoDateStamp(true);
+												tmpCreateQuery.setDisableAutoUserStamp(true);
+												tmpCreateQuery.setDisableDeleteTracking(true);
+												tmpCreateQuery.AllowIdentityInsert = true;
 
-												this.Meadow.doUpdate(tmpUpdateQuery,
-													(pUpdateError) =>
+												this.Meadow.doCreate(tmpCreateQuery,
+													(pCreateError) =>
 													{
-														if (pUpdateError)
+														if (pCreateError)
 														{
-															this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+															this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
 														}
 														tmpProcessed++;
 														return setImmediate(fRecordComplete);
 													});
-											});
-									},
-									(pRecordSyncError) =>
-									{
-										return fPageComplete();
-									});
-							});
-					},
-					(pDeleteSyncError) =>
-					{
-						const tmpElapsed = Date.now() - tmpStartTime;
-						if (tmpTimeBudgetExhausted)
-						{
-							this.fable.log.info(`Delete sync time budget exhausted for ${this.EntitySchema.TableName} after ${tmpElapsed}ms (${tmpProcessed} of ${tmpDeletedCount} deleted records processed).`);
-						}
-						else
-						{
-							this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpProcessed} of ${tmpDeletedCount} deleted records processed in ${tmpElapsed}ms).`);
-						}
-						return fCallback();
-					});
+												return;
+											}
+
+											if (pRecord.Deleted == 1)
+											{
+												tmpProcessed++;
+												return setImmediate(fRecordComplete);
+											}
+
+											const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+											const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+											tmpUpdateQuery.setDisableAutoIdentity(true);
+											tmpUpdateQuery.setDisableAutoDateStamp(true);
+											tmpUpdateQuery.setDisableAutoUserStamp(true);
+											tmpUpdateQuery.setDisableDeleteTracking(true);
+
+											this.Meadow.doUpdate(tmpUpdateQuery,
+												(pUpdateError) =>
+												{
+													if (pUpdateError)
+													{
+														this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+													}
+													tmpProcessed++;
+													return setImmediate(fRecordComplete);
+												});
+										});
+								},
+								(pRecordSyncError) =>
+								{
+									// Page complete — fetch next page
+									return setImmediate(fFetchDeletedPage);
+								});
+						});
+				};
+
+				fFetchDeletedPage();
 			});
 	}