meadow-integration 1.0.6 → 1.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "meadow-integration",
-    "version": "1.0.6",
+    "version": "1.0.7",
     "description": "Meadow Data Integration",
     "bin": {
         "mdwint": "source/cli/Meadow-Integration-CLI-Run.js"

package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js

@@ -39,6 +39,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 
 		this.DefaultIdentifier = this.EntitySchema.MeadowSchema.DefaultIdentifier;
 		this.PageSize = this.options.PageSize || 100;
+		this.SyncDeletedRecords = this.options.SyncDeletedRecords || false;
 
 		this.Meadow = false;
 
@@ -139,6 +140,136 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 		return tmpRecordToCommit;
 	}
 
+	syncDeletedRecords(fCallback)
+	{
+		const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
+		if (!tmpDeletedColumn)
+		{
+			this.fable.log.info(`No Deleted column for ${this.EntitySchema.TableName}; skipping delete sync.`);
+			return fCallback();
+		}
+
+		this.fable.log.info(`Checking for deleted records on server for ${this.EntitySchema.TableName}...`);
+
+		// Get the count of deleted records from the server.
+		// The explicit FBV~Deleted~EQ~1 filter overrides foxhound's automatic Deleted=0 filter.
+		this.fable.MeadowCloneRestClient.getJSON(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`,
+			(pError, pResponse, pBody) =>
+			{
+				if (pError || !pBody || !pBody.hasOwnProperty('Count'))
+				{
+					this.fable.log.warn(`Could not get deleted record count for ${this.EntitySchema.TableName}; skipping delete sync.`);
+					return fCallback();
+				}
+
+				const tmpDeletedCount = pBody.Count;
+				if (tmpDeletedCount < 1)
+				{
+					this.fable.log.info(`No deleted records on server for ${this.EntitySchema.TableName}.`);
+					return fCallback();
+				}
+
+				this.fable.log.info(`Found ${tmpDeletedCount} deleted records on server for ${this.EntitySchema.TableName}; syncing deletions...`);
+
+				// Generate paginated URLs for deleted records
+				const tmpDeleteURLPartials = [];
+				for (let i = 0; i < tmpDeletedCount; i += this.PageSize)
+				{
+					tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
+				}
+
+				this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
+					(pURLPartial, fPageComplete) =>
+					{
+						this.fable.MeadowCloneRestClient.getJSON(pURLPartial,
+							(pDownloadError, pResponse, pBody) =>
+							{
+								if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
+								{
+									return fPageComplete();
+								}
+
+								this.fable.Utility.eachLimit(pBody, 5,
+									(pEntityRecord, fRecordComplete) =>
+									{
+										const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
+										if (!tmpRecordID || tmpRecordID < 1)
+										{
+											return fRecordComplete();
+										}
+
+										// Read local record with delete tracking disabled so we can see all records
+										const tmpQuery = this.Meadow.query;
+										tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
+										tmpQuery.setDisableDeleteTracking(true);
+
+										this.Meadow.doRead(tmpQuery,
+											(pReadError, pQuery, pRecord) =>
+											{
+												if (pReadError || !pRecord)
+												{
+													// Record doesn't exist locally -- create it as deleted
+													const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+													const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+													tmpCreateQuery.setDisableAutoIdentity(true);
+													tmpCreateQuery.setDisableAutoDateStamp(true);
+													tmpCreateQuery.setDisableAutoUserStamp(true);
+													tmpCreateQuery.setDisableDeleteTracking(true);
+													tmpCreateQuery.AllowIdentityInsert = true;
+
+													this.Meadow.doCreate(tmpCreateQuery,
+														(pCreateError) =>
+														{
+															if (pCreateError)
+															{
+																this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
+															}
+															return fRecordComplete();
+														});
+													return;
+												}
+
+												if (pRecord.Deleted == 1)
+												{
+													// Already marked deleted locally
+													return fRecordComplete();
+												}
+
+												// Record exists locally but is not deleted -- update it
+												const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+												const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+												tmpUpdateQuery.setDisableAutoIdentity(true);
+												tmpUpdateQuery.setDisableAutoDateStamp(true);
+												tmpUpdateQuery.setDisableAutoUserStamp(true);
+												tmpUpdateQuery.setDisableDeleteTracking(true);
+
+												this.Meadow.doUpdate(tmpUpdateQuery,
+													(pUpdateError) =>
+													{
+														if (pUpdateError)
+														{
+															this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+														}
+														return fRecordComplete();
+													});
+											});
+									},
+									(pRecordSyncError) =>
+									{
+										return fPageComplete();
+									});
+							});
+					},
+					(pDeleteSyncError) =>
+					{
+						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpDeletedCount} deleted records processed).`);
+						return fCallback();
+					});
+			});
+	}
+
 	sync(fCallback)
 	{
 		this.operation.createTimeStamp('EntityInitialSync');
@@ -356,7 +487,11 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 				if (pError)
 				{
 					this.fable.log.error(`Error performing sync ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
-					return fCallback();
+				}
+
+				if (this.SyncDeletedRecords)
+				{
+					return this.syncDeletedRecords(() => { return fCallback(); });
 				}
 
 				return fCallback();

package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js

@@ -39,6 +39,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 
 		this.DefaultIdentifier = this.EntitySchema.MeadowSchema.DefaultIdentifier;
 		this.PageSize = this.options.PageSize || 100;
+		this.SyncDeletedRecords = this.options.SyncDeletedRecords || false;
 
 		this.Meadow = false;
 
@@ -132,6 +133,136 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 		return tmpRecordToCommit;
 	}
 
+	syncDeletedRecords(fCallback)
+	{
+		const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
+		if (!tmpDeletedColumn)
+		{
+			this.fable.log.info(`No Deleted column for ${this.EntitySchema.TableName}; skipping delete sync.`);
+			return fCallback();
+		}
+
+		this.fable.log.info(`Checking for deleted records on server for ${this.EntitySchema.TableName}...`);
+
+		// Get the count of deleted records from the server.
+		// The explicit FBV~Deleted~EQ~1 filter overrides foxhound's automatic Deleted=0 filter.
+		this.fable.MeadowCloneRestClient.getJSON(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`,
+			(pError, pResponse, pBody) =>
+			{
+				if (pError || !pBody || !pBody.hasOwnProperty('Count'))
+				{
+					this.fable.log.warn(`Could not get deleted record count for ${this.EntitySchema.TableName}; skipping delete sync.`);
+					return fCallback();
+				}
+
+				const tmpDeletedCount = pBody.Count;
+				if (tmpDeletedCount < 1)
+				{
+					this.fable.log.info(`No deleted records on server for ${this.EntitySchema.TableName}.`);
+					return fCallback();
+				}
+
+				this.fable.log.info(`Found ${tmpDeletedCount} deleted records on server for ${this.EntitySchema.TableName}; syncing deletions...`);
+
+				// Generate paginated URLs for deleted records
+				const tmpDeleteURLPartials = [];
+				for (let i = 0; i < tmpDeletedCount; i += this.PageSize)
+				{
+					tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
+				}
+
+				this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
+					(pURLPartial, fPageComplete) =>
+					{
+						this.fable.MeadowCloneRestClient.getJSON(pURLPartial,
+							(pDownloadError, pResponse, pBody) =>
+							{
+								if (pDownloadError || !pBody || !Array.isArray(pBody) || pBody.length < 1)
+								{
+									return fPageComplete();
+								}
+
+								this.fable.Utility.eachLimit(pBody, 5,
+									(pEntityRecord, fRecordComplete) =>
+									{
+										const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
+										if (!tmpRecordID || tmpRecordID < 1)
+										{
+											return fRecordComplete();
+										}
+
+										// Read local record with delete tracking disabled so we can see all records
+										const tmpQuery = this.Meadow.query;
+										tmpQuery.addFilter(this.DefaultIdentifier, tmpRecordID);
+										tmpQuery.setDisableDeleteTracking(true);
+
+										this.Meadow.doRead(tmpQuery,
+											(pReadError, pQuery, pRecord) =>
+											{
+												if (pReadError || !pRecord)
+												{
+													// Record doesn't exist locally -- create it as deleted
+													const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+													const tmpCreateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+													tmpCreateQuery.setDisableAutoIdentity(true);
+													tmpCreateQuery.setDisableAutoDateStamp(true);
+													tmpCreateQuery.setDisableAutoUserStamp(true);
+													tmpCreateQuery.setDisableDeleteTracking(true);
+													tmpCreateQuery.AllowIdentityInsert = true;
+
+													this.Meadow.doCreate(tmpCreateQuery,
+														(pCreateError) =>
+														{
+															if (pCreateError)
+															{
+																this.log.error(`Error creating deleted record ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pCreateError}`);
+															}
+															return fRecordComplete();
+														});
+													return;
+												}
+
+												if (pRecord.Deleted == 1)
+												{
+													// Already marked deleted locally
+													return fRecordComplete();
+												}
+
+												// Record exists locally but is not deleted -- update it
+												const tmpRecordToCommit = this.marshalRecord(pEntityRecord);
+
+												const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+												tmpUpdateQuery.setDisableAutoIdentity(true);
+												tmpUpdateQuery.setDisableAutoDateStamp(true);
+												tmpUpdateQuery.setDisableAutoUserStamp(true);
+												tmpUpdateQuery.setDisableDeleteTracking(true);
+
+												this.Meadow.doUpdate(tmpUpdateQuery,
+													(pUpdateError) =>
+													{
+														if (pUpdateError)
+														{
+															this.log.error(`Error marking record as deleted ${this.EntitySchema.TableName} ID ${tmpRecordID}: ${pUpdateError}`);
+														}
+														return fRecordComplete();
+													});
+											});
+									},
+									(pRecordSyncError) =>
+									{
+										return fPageComplete();
+									});
+							});
+					},
+					(pDeleteSyncError) =>
+					{
+						this.fable.log.info(`Delete sync complete for ${this.EntitySchema.TableName} (${tmpDeletedCount} deleted records processed).`);
+						return fCallback();
+					});
+			});
+	}
+
 	addSyncAnticipateEntry(tmpSyncState, tmpAnticipate)
 	{
 		tmpAnticipate.anticipate(
@@ -446,7 +577,11 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 				if (pError)
 				{
 					this.fable.log.error(`Error performing Update sync ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
-					return fCallback();
+				}
+
+				if (this.SyncDeletedRecords)
+				{
+					return this.syncDeletedRecords(() => { return fCallback(); });
 				}
 
 				return fCallback();

package/source/services/clone/Meadow-Service-Sync.js

@@ -43,6 +43,17 @@ class MeadowSync extends libFableServiceProviderBase
 			this.SyncEntityOptions = JSON.parse(JSON.stringify(this.options.SyncEntityOptions));
 		}
 
+		// When true, after syncing active records, also sync records marked Deleted=1 on the source.
+		this.SyncDeletedRecords = false;
+		if (this.fable.ProgramConfiguration.hasOwnProperty('SyncDeletedRecords'))
+		{
+			this.SyncDeletedRecords = !!this.fable.ProgramConfiguration.SyncDeletedRecords;
+		}
+		else if (this.options.hasOwnProperty('SyncDeletedRecords'))
+		{
+			this.SyncDeletedRecords = !!this.options.SyncDeletedRecords;
+		}
+
 		this.MeadowSchema = false;
 		this.MeadowSchemaTableList = false;
 
@@ -70,6 +81,7 @@ class MeadowSync extends libFableServiceProviderBase
 						MeadowEntitySchema: tmpEntitySchema,
 						ConnectionPool: this.options.ConnectionPool,
 						PageSize: this.options.PageSize || 100,
+						SyncDeletedRecords: this.SyncDeletedRecords,
 					};
 
 					let tmpSyncEntity;