meadow-integration 1.0.6 → 1.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "meadow-integration",
-    "version": "1.0.6",
+    "version": "1.0.9",
     "description": "Meadow Data Integration",
     "bin": {
         "mdwint": "source/cli/Meadow-Integration-CLI-Run.js"
@@ -16,7 +16,7 @@
     "author": "steven velozo <steven@velozo.com>",
     "license": "MIT",
     "devDependencies": {
-        "quackage": "^1.0.61"
+        "quackage": "^1.0.63"
     },
     "mocha": {
         "diff": true,
@@ -39,9 +39,9 @@
     "dependencies": {
         "fable": "^3.1.63",
         "fable-serviceproviderbase": "^3.0.19",
-        "meadow": "^2.0.28",
-        "meadow-connection-mysql": "^1.0.13",
-        "meadow-connection-mssql": "^1.0.15",
+        "meadow": "^2.0.30",
+        "meadow-connection-mysql": "^1.0.14",
+        "meadow-connection-mssql": "^1.0.16",
         "orator": "^6.0.4",
         "orator-serviceserver-restify": "^2.0.9",
         "pict-service-commandlineutility": "^1.0.19"

package/source/services/clone/Meadow-Service-RestClient.js

@@ -6,6 +6,14 @@ const defaultRestClientOptions = (
 	{
 		DownloadBatchSize: 100,
 
+		// Request timeout in milliseconds for normal remote API calls.
+		// Default: 60 seconds.
+		RequestTimeout: 60000,
+
+		// Request timeout in milliseconds for MAX(column) queries,
+		// which can be very slow on large tables. Default: 5 minutes.
+		MaxRequestTimeout: 300000,
+
 		ServerURL: 'https://localhost:8080/1.0/',
 		UserID: false,
 		Password: false,
@@ -36,7 +44,12 @@ class MeadowCloneRestClient extends libFableServiceProviderBase
 		this.restClient = this.fable.serviceManager.instantiateServiceProvider('RestClient', {}, 'MeadowCloneRestClient-RestClient');
 		this.cache = {};
 
-		const agentOptions = { keepAlive: true };
+		this.requestTimeout = this.options.RequestTimeout;
+		this.maxRequestTimeout = this.options.MaxRequestTimeout;
+
+		// Use the longer of the two timeouts for the agent's socket timeout
+		// so that MAX queries don't get killed at the socket level.
+		const agentOptions = { keepAlive: true, timeout: Math.max(this.requestTimeout, this.maxRequestTimeout) };
 
 		if (this.serverURL && this.serverURL.startsWith('http:'))
 		{

package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js

@@ -39,6 +39,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 
 		this.DefaultIdentifier = this.EntitySchema.MeadowSchema.DefaultIdentifier;
 		this.PageSize = this.options.PageSize || 100;
+		this.SyncDeletedRecords = this.options.SyncDeletedRecords || false;
 
 		this.Meadow = false;
 
@@ -52,12 +53,29 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 			this.Meadow = this.fable.Meadow.loadFromPackageObject(this.EntitySchema.MeadowSchema);
 		}
 
-		this.log.info(`Sync for ${this.EntitySchema.TableName} creating table if it doesn't exist...`);
-
 		if (this.Meadow && this.Meadow.provider)
 		{
-			return this.Meadow.provider.getProvider().createTable(this.EntitySchema, (pCreateError) =>
+			let tmpProvider = this.Meadow.provider.getProvider();
+
+			if (!tmpProvider)
+			{
+				this.log.error(`No provider returned by getProvider() for ${this.EntitySchema.TableName}`);
+				return fCallback(new Error(`No provider returned by getProvider() for ${this.EntitySchema.TableName}`));
+			}
+
+			if (!tmpProvider.createTable)
 			{
+				this.log.error(`Provider for ${this.EntitySchema.TableName} has no createTable method.`);
+				return fCallback(new Error(`Provider for ${this.EntitySchema.TableName} has no createTable method`));
+			}
+
+			return tmpProvider.createTable(this.EntitySchema, (pCreateError) =>
+			{
+				if (pCreateError)
+				{
+					this.log.warn(`${this.EntitySchema.TableName}: createTable returned error: ${pCreateError}`);
+				}
+
 				const tmpGUIDColumn = this.EntitySchema.Columns.find((c) => c.DataType == 'GUID');
 				const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
 
@@ -90,9 +108,17 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 							return this.fable.MeadowConnectionManager.createIndex(this.EntitySchema, tmpDeletedColumn, false, fNext);
 						});
 				}
-				tmpAnticipate.wait(fCallback);
+				tmpAnticipate.wait((pIndexError) =>
+				{
+					if (pIndexError)
+					{
+						this.log.warn(`${this.EntitySchema.TableName}: Index creation error: ${pIndexError}`);
+					}
+					return fCallback(pIndexError || pCreateError);
+				});
 			});
 		}
+
 		return fCallback();
 	}
 
@@ -139,10 +165,142 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 		return tmpRecordToCommit;
 	}
 
+	syncDeletedRecords(fCallback)
+	{
+		const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
+		if (!tmpDeletedColumn)
+		{
+			this.fable.log.info(`No Deleted column for ${this.EntitySchema.TableName}; skipping delete sync.`);
+			return fCallback();
+		}
+
+		this.fable.log.info(`Checking for deleted records on server for ${this.EntitySchema.TableName}...`);
+
+		// Get the count of deleted records from the server.
+		// The explicit FBV~Deleted~EQ~1 filter overrides foxhound's automatic Deleted=0 filter.
+		this.fable.MeadowCloneRestClient.getJSON(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`,
+			(pError, pResponse, pBody) =>
+			{
+				if (pError || !pBody || !pBody.hasOwnProperty('Count'))
+				{
+					this.fable.log.warn(`Could not get deleted record count for ${this.EntitySchema.TableName}; skipping delete sync.`);
+					return fCallback();
+				}
+
+				const tmpDeletedCount = pBody.Count;
+				if (tmpDeletedCount < 1)
+				{
+					this.fable.log.info(`No deleted records on server for ${this.EntitySchema.TableName}.`);
+					return fCallback();
+				}
+
+				this.fable.log.info(`Found ${tmpDeletedCount} deleted records on server for ${this.EntitySchema.TableName}; syncing deletions...`);
+
+				// Generate paginated URLs for deleted records
+				const tmpDeleteURLPartials = [];
+				for (let i = 0; i < tmpDeletedCount; i += this.PageSize)
+				{
+					tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
+				}
+
+				this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
+					(pURLPartial, fPageComplete) =>
+					{
+						this.fable.MeadowCloneRestClient.getJSON(pURLPartial,
+							(pDownloadError, pResponse, pBody) =>
+							{
+								if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
+								{
+									return fPageComplete();
+								}
+
+								this.fable.Utility.eachLimit(pBody, 5,
+									(pEntityRecord, fRecordComplete) =>
+									{
+										const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
+										if (!tmpRecordID || tmpRecordID < 1)
+										{
+											return fRecordComplete();
+										}
+
+										// Read local record with delete tracking disabled so we can see all records
+										const tmpQuery = this.Meadow.query;
+										tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
+										tmpQuery.setDisableDeleteTracking(true);
+
+										this.Meadow.doRead(tmpQuery,
+											(pReadError, pQuery, pRecord) =>
+											{
+												if (pReadError || !pRecord)
+												{
+													// Record doesn't exist locally -- create it as deleted
+													const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+													const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+													tmpCreateQuery.setDisableAutoIdentity(true);
+													tmpCreateQuery.setDisableAutoDateStamp(true);
+													tmpCreateQuery.setDisableAutoUserStamp(true);
+													tmpCreateQuery.setDisableDeleteTracking(true);
+													tmpCreateQuery.AllowIdentityInsert = true;
+
+													this.Meadow.doCreate(tmpCreateQuery,
+														(pCreateError) =>
+														{
+															if (pCreateError)
+															{
+																this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
+															}
+															return fRecordComplete();
+														});
+													return;
+												}
+
+												if (pRecord.Deleted == 1)
+												{
+													// Already marked deleted locally
+													return fRecordComplete();
+												}
+
+												// Record exists locally but is not deleted -- update it
+												const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+												const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+												tmpUpdateQuery.setDisableAutoIdentity(true);
+												tmpUpdateQuery.setDisableAutoDateStamp(true);
+												tmpUpdateQuery.setDisableAutoUserStamp(true);
+												tmpUpdateQuery.setDisableDeleteTracking(true);
+
+												this.Meadow.doUpdate(tmpUpdateQuery,
+													(pUpdateError) =>
+													{
+														if (pUpdateError)
+														{
+															this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+														}
+														return fRecordComplete();
+													});
+											});
+									},
+									(pRecordSyncError) =>
+									{
+										return fPageComplete();
+									});
+							});
+					},
+					(pDeleteSyncError) =>
+					{
+						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpDeletedCount} deleted records processed).`);
+						return fCallback();
+					});
+			});
+	}
+
 	sync(fCallback)
 	{
 		this.operation.createTimeStamp('EntityInitialSync');
 
+		this.log.info(`Syncing ${this.EntitySchema.TableName} (PageSize: ${this.PageSize}, SyncDeletedRecords: ${this.SyncDeletedRecords})`);
+
 		const tmpSyncState = (
 			{
 				Local: { MaxIDEntity: -1, RecordCount: 0 },
@@ -162,16 +320,12 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 						{
 							if (pReadError)
 							{
-								this.fable.log.error(`Error reading local max entity ID ${this.EntitySchema.TableName}: ${pReadError}`, { Error: pReadError });
 								return fStageComplete(`Error reading local max entity ID ${this.EntitySchema.TableName}: ${pReadError}`);
 							}
-							if (!pRecord)
+							if (pRecord)
 							{
-								this.fable.log.warn(`No records found in local ${this.EntitySchema.TableName}.`);
-								return fStageComplete();
+								tmpSyncState.Local.MaxIDEntity = pRecord[this.DefaultIdentifier];
 							}
-							this.fable.log.info(`Found local max entity ID ${this.EntitySchema.TableName}: ${pRecord[this.DefaultIdentifier]}`);
-							tmpSyncState.Local.MaxIDEntity = pRecord[this.DefaultIdentifier];
 							return fStageComplete();
 						});
 				},
@@ -184,7 +338,6 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 						{
 							if (pCountError)
 							{
-								this.fable.log.error(`Error getting local count of ${this.EntitySchema.TableName}: ${pCountError}`, { Error: pCountError });
 								return fStageComplete(`Error getting local count of ${this.EntitySchema.TableName}: ${pCountError}`);
 							}
 							tmpSyncState.Local.RecordCount = pCount;
@@ -199,18 +352,12 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 						{
 							if (pError)
 							{
-								this.fable.log.error(`Error getting server max entity ID ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
 								return fStageComplete(`Error getting server max entity ID ${this.EntitySchema.TableName}: ${pError}`);
 							}
 							if (pBody && pBody.hasOwnProperty(this.DefaultIdentifier))
 							{
-								this.fable.log.info(`Found server max entity ID ${this.EntitySchema.TableName}: ${pBody[this.DefaultIdentifier]}`);
 								tmpSyncState.Server.MaxIDEntity = pBody[this.DefaultIdentifier];
 							}
-							else
-							{
-								this.fable.log.warn(`No records found in server for max entity ID of ${this.EntitySchema.TableName}.`);
-							}
 							return fStageComplete();
 						});
 				},
@@ -222,18 +369,12 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 						{
 							if (pError)
 							{
-								this.fable.log.error(`Error getting server count for ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
 								return fStageComplete(`Error getting server count for ${this.EntitySchema.TableName}: ${pError}`);
 							}
 							if (pBody && pBody.hasOwnProperty('Count'))
 							{
-								this.fable.log.info(`Found server count for ${this.EntitySchema.TableName}: ${pBody.Count}`);
 								tmpSyncState.Server.RecordCount = pBody.Count;
 							}
-							else
-							{
-								this.fable.log.warn(`No records found in server based on count for ${this.EntitySchema.TableName}.`);
-							}
 							return fStageComplete();
 						});
 				},
@@ -251,21 +392,28 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 						tmpSyncState.URLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~${this.DefaultIdentifier}~GT~${tmpSyncState.Local.MaxIDEntity}~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
 					}
 
-					this.fable.log.info(`Syncing with ${tmpSyncState.URLPartials.length} requests for ${this.EntitySchema.TableName} with local max ID ${tmpSyncState.Local.MaxIDEntity} and server max ID ${tmpSyncState.Server.MaxIDEntity}; estimated ${tmpSyncState.EstimatedRecordCount} records to sync.`);
+					this.fable.log.info(`${this.EntitySchema.TableName}: downloading ${tmpSyncState.URLPartials.length} pages (local: ${tmpSyncState.Local.RecordCount}/${tmpSyncState.Local.MaxIDEntity}, server: ${tmpSyncState.Server.RecordCount}/${tmpSyncState.Server.MaxIDEntity}, estimated new: ${tmpSyncState.EstimatedRecordCount})`);
 
 					return fStageComplete();
 				},
 				(fStageComplete) =>
 				{
+					let tmpPageIndex = 0;
+					let tmpRecordsCreated = 0;
+					let tmpRecordsSkipped = 0;
+					let tmpRecordsErrored = 0;
+
 					this.fable.Utility.eachLimit(tmpSyncState.URLPartials, 1,
 						(pURLPartial, fDownloadComplete) =>
 						{
+							tmpPageIndex++;
+
 							this.fable.MeadowCloneRestClient.getJSON(pURLPartial,
 								(pDownloadError, pResponse, pBody) =>
 								{
 									if (pDownloadError)
 									{
-										this.fable.log.error(`Error getting URL Partial [${pURLPartial}]: ${pDownloadError}`, { Error: pDownloadError });
+										this.fable.log.error(`${this.EntitySchema.TableName}: page ${tmpPageIndex} download error: ${pDownloadError}`);
 										return fDownloadComplete();
 									}
 									if (pBody && pBody.length > 0)
@@ -286,7 +434,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 													{
 														if (pReadError)
 														{
-															this.fable.log.error(`Error reading record ${this.EntitySchema.TableName}: ${pReadError}`, { Error: pReadError, PassedRecord: tmpRecord });
+															tmpRecordsErrored++;
 															return fEntitySyncComplete();
 														}
 														if (!pRecord)
@@ -308,15 +456,18 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 																{
 																	if (pCreateError)
 																	{
-																		this.log.error(`Error creating record ${this.EntitySchema.TableName}: ${pCreateError}`, pCreateError);
+																		tmpRecordsErrored++;
+																		this.log.error(`${this.EntitySchema.TableName}: doCreate error for ID ${tmpRecord[this.DefaultIdentifier]}: ${pCreateError}`);
 																		return fEntitySyncComplete();
 																	}
+																	tmpRecordsCreated++;
 																	this.operation.incrementProgressTrackerStatus(`FullSync-${this.EntitySchema.TableName}`, 1);
 																	return fEntitySyncComplete();
 																});
 														}
 														else
 														{
+															tmpRecordsSkipped++;
 															return fEntitySyncComplete();
 														}
 													});
@@ -343,6 +494,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 						},
 						(pDownloadError) =>
 						{
+							this.fable.log.info(`${this.EntitySchema.TableName}: sync complete — created: ${tmpRecordsCreated}, skipped: ${tmpRecordsSkipped}, errors: ${tmpRecordsErrored}`);
 							if (pDownloadError)
 							{
 								this.fable.log.error(`Error returned URL Partial .. this may not be an error: ${pDownloadError}`);
@@ -355,8 +507,12 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 			{
 				if (pError)
 				{
-					this.fable.log.error(`Error performing sync ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
-					return fCallback();
+					this.fable.log.error(`${this.EntitySchema.TableName}: sync error: ${pError}`);
+				}
+
+				if (this.SyncDeletedRecords)
+				{
+					return this.syncDeletedRecords(() => { return fCallback(); });
 				}
 
 				return fCallback();

package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js

@@ -39,6 +39,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 
 		this.DefaultIdentifier = this.EntitySchema.MeadowSchema.DefaultIdentifier;
 		this.PageSize = this.options.PageSize || 100;
+		this.SyncDeletedRecords = this.options.SyncDeletedRecords || false;
 
 		this.Meadow = false;
 
@@ -132,6 +133,136 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 		return tmpRecordToCommit;
 	}
 
+	syncDeletedRecords(fCallback)
+	{
+		const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
+		if (!tmpDeletedColumn)
+		{
+			this.fable.log.info(`No Deleted column for ${this.EntitySchema.TableName}; skipping delete sync.`);
+			return fCallback();
+		}
+
+		this.fable.log.info(`Checking for deleted records on server for ${this.EntitySchema.TableName}...`);
+
+		// Get the count of deleted records from the server.
+		// The explicit FBV~Deleted~EQ~1 filter overrides foxhound's automatic Deleted=0 filter.
+		this.fable.MeadowCloneRestClient.getJSON(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`,
+			(pError, pResponse, pBody) =>
+			{
+				if (pError || !pBody || !pBody.hasOwnProperty('Count'))
+				{
+					this.fable.log.warn(`Could not get deleted record count for ${this.EntitySchema.TableName}; skipping delete sync.`);
+					return fCallback();
+				}
+
+				const tmpDeletedCount = pBody.Count;
+				if (tmpDeletedCount < 1)
+				{
+					this.fable.log.info(`No deleted records on server for ${this.EntitySchema.TableName}.`);
+					return fCallback();
+				}
+
+				this.fable.log.info(`Found ${tmpDeletedCount} deleted records on server for ${this.EntitySchema.TableName}; syncing deletions...`);
+
+				// Generate paginated URLs for deleted records
+				const tmpDeleteURLPartials = [];
+				for (let i = 0; i < tmpDeletedCount; i += this.PageSize)
+				{
+					tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
+				}
+
+				this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
+					(pURLPartial, fPageComplete) =>
+					{
+						this.fable.MeadowCloneRestClient.getJSON(pURLPartial,
+							(pDownloadError, pResponse, pBody) =>
+							{
+								if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
+								{
+									return fPageComplete();
+								}
+
+								this.fable.Utility.eachLimit(pBody, 5,
+									(pEntityRecord, fRecordComplete) =>
+									{
+										const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
+										if (!tmpRecordID || tmpRecordID < 1)
+										{
+											return fRecordComplete();
+										}
+
+										// Read local record with delete tracking disabled so we can see all records
+										const tmpQuery = this.Meadow.query;
+										tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
+										tmpQuery.setDisableDeleteTracking(true);
+
+										this.Meadow.doRead(tmpQuery,
+											(pReadError, pQuery, pRecord) =>
+											{
+												if (pReadError || !pRecord)
+												{
+													// Record doesn't exist locally -- create it as deleted
+													const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+													const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+													tmpCreateQuery.setDisableAutoIdentity(true);
+													tmpCreateQuery.setDisableAutoDateStamp(true);
+													tmpCreateQuery.setDisableAutoUserStamp(true);
+													tmpCreateQuery.setDisableDeleteTracking(true);
+													tmpCreateQuery.AllowIdentityInsert = true;
+
+													this.Meadow.doCreate(tmpCreateQuery,
+														(pCreateError) =>
+														{
+															if (pCreateError)
+															{
+																this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
+															}
+															return fRecordComplete();
+														});
+													return;
+												}
+
+												if (pRecord.Deleted == 1)
+												{
+													// Already marked deleted locally
+													return fRecordComplete();
+												}
+
+												// Record exists locally but is not deleted -- update it
+												const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+												const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+												tmpUpdateQuery.setDisableAutoIdentity(true);
+												tmpUpdateQuery.setDisableAutoDateStamp(true);
+												tmpUpdateQuery.setDisableAutoUserStamp(true);
+												tmpUpdateQuery.setDisableDeleteTracking(true);
+
+												this.Meadow.doUpdate(tmpUpdateQuery,
+													(pUpdateError) =>
+													{
+														if (pUpdateError)
+														{
+															this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+														}
+														return fRecordComplete();
+													});
+											});
+									},
+									(pRecordSyncError) =>
+									{
+										return fPageComplete();
+									});
+							});
+					},
+					(pDeleteSyncError) =>
+					{
+						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpDeletedCount} deleted records processed).`);
+						return fCallback();
+					});
+			});
+	}
+
 	addSyncAnticipateEntry(tmpSyncState, tmpAnticipate)
 	{
 		tmpAnticipate.anticipate(
@@ -446,7 +577,11 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 				if (pError)
 				{
 					this.fable.log.error(`Error performing Update sync ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
-					return fCallback();
+				}
+
+				if (this.SyncDeletedRecords)
+				{
+					return this.syncDeletedRecords(() => { return fCallback(); });
 				}
 
 				return fCallback();

package/source/services/clone/Meadow-Service-Sync.js

@@ -43,6 +43,17 @@ class MeadowSync extends libFableServiceProviderBase
 			this.SyncEntityOptions = JSON.parse(JSON.stringify(this.options.SyncEntityOptions));
 		}
 
+		// When true, after syncing active records, also sync records marked Deleted=1 on the source.
+		this.SyncDeletedRecords = false;
+		if (this.fable.ProgramConfiguration.hasOwnProperty('SyncDeletedRecords'))
+		{
+			this.SyncDeletedRecords = !!this.fable.ProgramConfiguration.SyncDeletedRecords;
+		}
+		else if (this.options.hasOwnProperty('SyncDeletedRecords'))
+		{
+			this.SyncDeletedRecords = !!this.options.SyncDeletedRecords;
+		}
+
 		this.MeadowSchema = false;
 		this.MeadowSchemaTableList = false;
 
@@ -59,9 +70,16 @@ class MeadowSync extends libFableServiceProviderBase
 		this.meadowSchema = pSchema;
 		this.MeadowSchemaTableList = Object.keys(this.meadowSchema.Tables);
 
+		this.log.info(`Loading schema for ${this.MeadowSchemaTableList.length} tables (mode: ${this.SyncMode})`);
+
+		let tmpEntityIndex = 0;
+		let tmpErrorCount = 0;
+		let tmpSuccessCount = 0;
+
 		this.fable.Utility.eachLimit(this.MeadowSchemaTableList, 1,
 			(pEntitySchemaName, fSyncInitializationComplete) =>
 			{
+				tmpEntityIndex++;
 				const tmpEntitySchema = this.meadowSchema.Tables[pEntitySchemaName];
 				// If this is in the entity list or none is specified, create the sync entity object.
 				if (this.SyncEntityList.length < 1 || this.SyncEntityList.indexOf(tmpEntitySchema.TableName) > -1)
@@ -70,6 +88,7 @@ class MeadowSync extends libFableServiceProviderBase
 						MeadowEntitySchema: tmpEntitySchema,
 						ConnectionPool: this.options.ConnectionPool,
 						PageSize: this.options.PageSize || 100,
+						SyncDeletedRecords: this.SyncDeletedRecords,
 					};
 
 					let tmpSyncEntity;
@@ -85,7 +104,20 @@ class MeadowSync extends libFableServiceProviderBase
 
 					this.MeadowSyncEntities[tmpEntitySchema.TableName] = tmpSyncEntity;
 
-					return tmpSyncEntity.initialize(fSyncInitializationComplete);
+					return tmpSyncEntity.initialize((pInitError) =>
+					{
+						if (pInitError)
+						{
+							tmpErrorCount++;
+							this.log.warn(`Failed to initialize ${tmpEntitySchema.TableName}: ${pInitError}`);
+						}
+						else
+						{
+							tmpSuccessCount++;
+						}
+						// Always continue to next entity regardless of individual errors
+						return fSyncInitializationComplete();
+					});
 				}
 				else
 				{
@@ -99,7 +131,7 @@ class MeadowSync extends libFableServiceProviderBase
 					this.log.error(`MeadowSync Error creating sync objects: ${pSyncInitializationError}`, pSyncInitializationError);
 				}
 
-				this.log.info('Entity sync objects created!');
+				this.log.info(`Entity sync objects created: ${tmpSuccessCount} succeeded, ${tmpErrorCount} failed.`);
 
 				if (this.SyncEntityList.length < 1)
 				{
@@ -117,7 +149,15 @@ class MeadowSync extends libFableServiceProviderBase
 			this.log.warn(`MeadowSync.syncEntity called for an entity that does not exist: ${pEntityHash}`);
 			return fCallback();
 		}
-		this.MeadowSyncEntities[pEntityHash].sync(fCallback);
+
+		this.MeadowSyncEntities[pEntityHash].sync((pError) =>
+		{
+			if (pError)
+			{
+				this.log.error(`Sync failed for ${pEntityHash}: ${pError}`);
+			}
+			return fCallback(pError);
+		});
 	}
 
 	syncAll(fCallback)