meadow-integration 1.0.14 → 1.0.16

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "meadow-integration",
-    "version": "1.0.14",
+    "version": "1.0.16",
     "description": "Meadow Data Integration",
     "bin": {
         "mdwint": "source/cli/Meadow-Integration-CLI-Run.js"
@@ -39,7 +39,7 @@
     "dependencies": {
         "fable": "^3.1.63",
         "fable-serviceproviderbase": "^3.0.19",
-        "meadow": "^2.0.30",
+        "meadow": "^2.0.33",
         "meadow-connection-mssql": "^1.0.16",
         "meadow-connection-mysql": "^1.0.14",
         "orator": "^6.0.4",

package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js

@@ -297,6 +297,10 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 					(pDeleteSyncError) =>
 					{
 						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpDeletedCount} deleted records processed).`);
+						if (this.syncResults)
+						{
+							this.syncResults.Deleted = tmpDeletedCount;
+						}
 						return fCallback();
 					});
 			});
@@ -599,6 +603,21 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 							{
 								this.fable.log.error(`Error returned URL Partial .. this may not be an error: ${pDownloadError}`);
 							}
+
+							// Store sync results on the entity so callers can inspect the breakdown
+							this.syncResults = (
+								{
+									Created: tmpRecordsCreated,
+									Skipped: tmpRecordsSkipped,
+									Errors: tmpRecordsErrored,
+									Deleted: 0,
+									ServerRecordCount: tmpSyncState.Server.RecordCount,
+									LocalRecordCount: tmpSyncState.Local.RecordCount,
+									ServerMaxID: tmpSyncState.Server.MaxIDEntity,
+									LocalMaxID: tmpSyncState.Local.MaxIDEntity,
+									EstimatedNew: tmpSyncState.EstimatedRecordCount
+								});
+
 							fStageComplete();
 						});
 				},

package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js

@@ -372,6 +372,10 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 											{
 												this.log.error(`Fallback update also failed for ${this.EntitySchema.TableName} ID ${pServerRecord[this.DefaultIdentifier]}: ${pUpdateError}`);
 											}
+											else
+											{
+												this._recordsUpdated++;
+											}
 											return fCallback();
 										});
 									return;
@@ -379,6 +383,10 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 								this.log.error(`Error creating record ${this.EntitySchema.TableName}: ${pCreateError}`, pCreateError);
 								return fCallback();
 							}
+							else
+							{
+								this._recordsCreated++;
+							}
 							return fCallback();
 						});
 				}
@@ -392,6 +400,10 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 							{
 								this.log.error(`Error updating record ${this.EntitySchema.TableName}: ${pUpdateError}`, pUpdateError);
 							}
+							else
+							{
+								this._recordsUpdated++;
+							}
 							return fCallback();
 						});
 				}
@@ -407,12 +419,22 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 			return fCallback(null, 0);
 		}
 
+		// Check the global cap before starting -- if we have already synced
+		// MaxRecordsPerEntity records across all stages, stop immediately.
+		if (this.MaxRecordsPerEntity > 0 && this._totalSyncedThisSync >= this.MaxRecordsPerEntity)
+		{
+			this.fable.log.info(`${this.EntitySchema.TableName}: global record cap reached (${this._totalSyncedThisSync}/${this.MaxRecordsPerEntity}); skipping pull.`);
+			return fCallback(null, 0);
+		}
+
 		let tmpSyncedCount = 0;
 		let tmpOffset = 0;
 		let tmpDone = false;
 
+		// Apply per-call cap based on estimated count, then further limit by
+		// how many records remain before hitting the global MaxRecordsPerEntity.
 		let tmpRecordCap = (this.MaxRecordsPerEntity > 0)
-			? Math.min(pEstimatedCount, this.MaxRecordsPerEntity)
+			? Math.min(pEstimatedCount, this.MaxRecordsPerEntity - this._totalSyncedThisSync)
 			: pEstimatedCount;
 
 		const fFetchPage = () =>
@@ -443,11 +465,16 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 								() =>
 								{
 									tmpSyncedCount++;
-									return fRecordDone();
+									this._totalSyncedThisSync++;
+									// Use setImmediate to yield the event loop and prevent
+									// stack overflow when SQLite callbacks complete synchronously
+									return setImmediate(fRecordDone);
 								});
 						},
 						(pUpsertError) =>
 						{
+							// Increment per-page progress so the UI reflects sync in real-time
+							this._incrementProgress(pRecords.length);
 							tmpOffset += this.PageSize;
 							if (pRecords.length < this.PageSize)
 							{
@@ -455,7 +482,8 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 								return fCallback(null, tmpSyncedCount);
 							}
 							this.fable.log.info(`${this.EntitySchema.TableName}: pulled ${tmpSyncedCount} of ~${tmpRecordCap} records...`);
-							return fFetchPage();
+							// Use setImmediate to break the recursive call chain across pages
+							return setImmediate(fFetchPage);
 						});
 				});
 		};
@@ -480,6 +508,12 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 	// the range from the server to bring local in sync.
 	_bisectRange(pMinID, pMaxID, pDepth, fCallback)
 	{
+		// If the global record cap has been reached, stop bisecting
+		if (this.MaxRecordsPerEntity > 0 && this._totalSyncedThisSync >= this.MaxRecordsPerEntity)
+		{
+			return fCallback();
+		}
+
 		const tmpRangeSize = pMaxID - pMinID + 1;
 		const tmpIDCol = this.DefaultIdentifier;
 		const tmpRangeFilter = `FBV~${tmpIDCol}~GE~${pMinID}~FBV~${tmpIDCol}~LE~${pMaxID}`;
@@ -604,7 +638,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 				{
 					this.fable.log.info(`${this.EntitySchema.TableName}: synced ${pSyncedCount} records in range ${pMinID}-${pMaxID}`);
 				}
-				this._incrementProgress(pSyncedCount || 0);
+				// Per-page progress is now tracked inside _pullServerRecords()
 				return fCallback();
 			});
 	}
@@ -763,7 +797,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 			{
 				if (pReadError)
 				{
-					let tmpErrorStr = (typeof(pReadError) === 'string') ? pReadError : JSON.stringify(pReadError);
+					let tmpErrorStr = (typeof(pReadError) === 'string') ? pReadError : String(pReadError);
 					if (tmpErrorStr.indexOf('Invalid column') > -1 || tmpErrorStr.indexOf('Invalid object') > -1 || tmpErrorStr.indexOf('no such column') > -1 || tmpErrorStr.indexOf('no such table') > -1)
 					{
 						this.log.warn(`${this.EntitySchema.TableName}: local table schema mismatch (${pReadError}); skipping sync.`);
@@ -778,6 +812,13 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 	{
 		this.operation.createTimeStamp('EntityOngoingSync');
 
+		// Track total records synced across all stages to enforce MaxRecordsPerEntity globally
+		this._totalSyncedThisSync = 0;
+
+		// Track per-record create vs update counts for the sync report
+		this._recordsCreated = 0;
+		this._recordsUpdated = 0;
+
 		const tmpSyncState = (
 			{
 				Local: { MaxIDEntity: -1, RecordCount: 0 },
@@ -936,7 +977,10 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 					// Create a progress tracker so callers (e.g. data-cloner UI) can see Total/Synced
 				(fStageComplete) =>
 				{
-					this.operation.createProgressTracker(tmpSyncState.Server.RecordCount, `FullSync-${this.EntitySchema.TableName}`);
+					let tmpTrackerTotal = (this.MaxRecordsPerEntity > 0)
+					? Math.min(tmpSyncState.Server.RecordCount, this.MaxRecordsPerEntity)
+					: tmpSyncState.Server.RecordCount;
+				this.operation.createProgressTracker(tmpTrackerTotal, `FullSync-${this.EntitySchema.TableName}`);
 					return fStageComplete();
 				},
 
@@ -1017,7 +1061,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 											{
 												this.fable.log.info(`${this.EntitySchema.TableName}: pulled ${pSyncedCount} new/modified records via UpdateDate.`);
 											}
-											this._incrementProgress(pSyncedCount || 0);
+											// Per-page progress is now tracked inside _pullServerRecords()
 											tmpSyncState.UpdateDateSyncDone = true;
 											return fStageComplete();
 										});
@@ -1100,7 +1144,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 							{
 								this.fable.log.info(`${this.EntitySchema.TableName}: pulled ${pSyncedCount} new records by ID.`);
 							}
-							this._incrementProgress(pSyncedCount || 0);
+							// Per-page progress is now tracked inside _pullServerRecords()
 							return fStageComplete();
 						});
 				},
@@ -1121,6 +1165,15 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 
 				this.fable.log.info(`${this.EntitySchema.TableName}: ongoing sync complete.`);
 
+		// Store sync results so callers can inspect the breakdown
+		this.syncResults = {
+			Created: this._recordsCreated,
+			Updated: this._recordsUpdated,
+			Deleted: 0,
+			ServerRecordCount: tmpSyncState.Server.RecordCount,
+			LocalRecordCount: tmpSyncState.Local.RecordCount
+		};
+
 				if (this.SyncDeletedRecords)
 				{
 					return this.syncDeletedRecords(() => { return fCallback(); });