meadow-integration 1.0.10 → 1.0.12

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "meadow-integration",
-    "version": "1.0.10",
+    "version": "1.0.12",
     "description": "Meadow Data Integration",
     "bin": {
         "mdwint": "source/cli/Meadow-Integration-CLI-Run.js"

package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js

@@ -45,6 +45,8 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 		this.Meadow = false;
 
 		this.operation = new libMeadowOperation(this.fable);
+
+		this.skipSync = false;
 	}
 
 	initialize(fCallback)
@@ -72,6 +74,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 
 			return tmpProvider.createTable(this.EntitySchema, (pCreateError) =>
 			{
+
 				if (pCreateError)
 				{
 					this.log.warn(`${this.EntitySchema.TableName}: createTable returned error: ${pCreateError}`);
@@ -115,7 +118,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 					{
 						this.log.warn(`${this.EntitySchema.TableName}: Index creation error: ${pIndexError}`);
 					}
-					return fCallback(pIndexError || pCreateError);
+					return fValidateAndCallback(pIndexError || pCreateError);
 				});
 			});
 		}
@@ -300,6 +303,35 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 	}
 
 	sync(fCallback)
+	{
+		if (this.skipSync)
+		{
+			this.log.warn(`Skipping sync for ${this.EntitySchema.TableName} -- local table schema does not match expected schema.`);
+			return fCallback();
+		}
+
+		// Validate local table schema with a lightweight read before syncing
+		const tmpValidationQuery = this.Meadow.query;
+		tmpValidationQuery.setSort({ Column: this.DefaultIdentifier, Direction: 'Descending' });
+		tmpValidationQuery.setCap(1);
+		tmpValidationQuery.setDisableDeleteTracking(true);
+		this.Meadow.doRead(tmpValidationQuery,
+			(pReadError) =>
+			{
+				if (pReadError)
+				{
+					let tmpErrorStr = (typeof(pReadError) === 'string') ? pReadError : JSON.stringify(pReadError);
+					if (tmpErrorStr.indexOf('Invalid column') > -1 || tmpErrorStr.indexOf('Invalid object') > -1 || tmpErrorStr.indexOf('no such column') > -1 || tmpErrorStr.indexOf('no such table') > -1)
+					{
+						this.log.warn(`${this.EntitySchema.TableName}: local table schema mismatch (${pReadError}); skipping sync.`);
+						return fCallback();
+					}
+				}
+				return this._syncInternal(fCallback);
+			});
+	}
+
+	_syncInternal(fCallback)
 	{
 		this.operation.createTimeStamp('EntityInitialSync');
 
@@ -377,7 +409,8 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 						{
 							if (pError)
 							{
-								return fStageComplete(`Error getting server max entity ID ${this.EntitySchema.TableName}: ${pError}`);
+								this.fable.log.warn(`Could not get server max entity ID for ${this.EntitySchema.TableName} (${pError}); continuing sync.`);
+								return fStageComplete();
 							}
 							if (pBody && pBody.hasOwnProperty(this.DefaultIdentifier))
 							{
@@ -394,7 +427,9 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 						{
 							if (pError)
 							{
-								return fStageComplete(`Error getting server count for ${this.EntitySchema.TableName}: ${pError}`);
+								this.fable.log.warn(`Could not get server count for ${this.EntitySchema.TableName} (${pError}); estimating from max ID.`);
+								tmpSyncState.Server.RecordCount = tmpSyncState.Server.MaxIDEntity > 0 ? tmpSyncState.Server.MaxIDEntity : 0;
+								return fStageComplete();
 							}
 							if (pBody && pBody.hasOwnProperty('Count'))
 							{
@@ -451,7 +486,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 										this.fable.log.error(`${this.EntitySchema.TableName}: page ${tmpPageIndex} download error: ${pDownloadError}`);
 										return fDownloadComplete();
 									}
-									if (pBody && pBody.length > 0)
+									if (pBody && Array.isArray(pBody) && pBody.length > 0)
 									{
 										this.fable.Utility.eachLimit(pBody, 5,
 											(pEntityRecord, fEntitySyncComplete) =>

package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js

@@ -45,6 +45,8 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 		this.Meadow = false;
 
 		this.operation = new libMeadowOperation(this.fable);
+
+		this.skipSync = false;
 	}
 
 	initialize(fCallback)
@@ -60,6 +62,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 		{
 			return this.Meadow.provider.getProvider().createTable(this.EntitySchema, (pCreateError) =>
 			{
+
 				const tmpGUIDColumn = this.EntitySchema.Columns.find((c) => c.DataType == 'GUID');
 				const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
 
@@ -92,7 +95,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 							return this.fable.MeadowConnectionManager.createIndex(this.EntitySchema, tmpDeletedColumn, false, fNext);
 						});
 				}
-				tmpAnticipate.wait(fCallback);
+				tmpAnticipate.wait((pIndexError) => { return fCallback(pCreateError); });
 			});
 		}
 		return fCallback();
@@ -288,7 +291,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 							this.fable.log.error(`Error getting URL Partial [${tmpURLPartial}]: ${pDownloadError}`, { Error: pDownloadError });
 							return fNext();
 						}
-						if (pBody && pBody.length > 0)
+						if (pBody && Array.isArray(pBody) && pBody.length > 0)
 						{
 							for (let i = 0; i < pBody.length; i++)
 							{
@@ -419,6 +422,35 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 	}
 
 	sync(fCallback)
+	{
+		if (this.skipSync)
+		{
+			this.log.warn(`Skipping sync for ${this.EntitySchema.TableName} -- local table schema does not match expected schema.`);
+			return fCallback();
+		}
+
+		// Validate local table schema with a lightweight read before syncing
+		const tmpValidationQuery = this.Meadow.query;
+		tmpValidationQuery.setSort({ Column: this.DefaultIdentifier, Direction: 'Descending' });
+		tmpValidationQuery.setCap(1);
+		tmpValidationQuery.setDisableDeleteTracking(true);
+		this.Meadow.doRead(tmpValidationQuery,
+			(pReadError) =>
+			{
+				if (pReadError)
+				{
+					let tmpErrorStr = (typeof(pReadError) === 'string') ? pReadError : JSON.stringify(pReadError);
+					if (tmpErrorStr.indexOf('Invalid column') > -1 || tmpErrorStr.indexOf('Invalid object') > -1 || tmpErrorStr.indexOf('no such column') > -1 || tmpErrorStr.indexOf('no such table') > -1)
+					{
+						this.log.warn(`${this.EntitySchema.TableName}: local table schema mismatch (${pReadError}); skipping sync.`);
+						return fCallback();
+					}
+				}
+				return this._syncInternal(fCallback);
+			});
+	}
+
+	_syncInternal(fCallback)
 	{
 		this.operation.createTimeStamp('EntityOngoingSync');
 
@@ -551,8 +583,8 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 						{
 							if (pError)
 							{
-								this.fable.log.error(`Error getting server max entity ID ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
-								return fStageComplete(`Error getting server max entity ID ${this.EntitySchema.TableName}: ${pError}`);
+								this.fable.log.warn(`Could not get server max entity ID for ${this.EntitySchema.TableName} (${pError}); continuing sync.`);
+								return fStageComplete();
 							}
 							if (pBody && pBody.hasOwnProperty(this.DefaultIdentifier))
 							{
@@ -574,8 +606,8 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 						{
 							if (pError)
 							{
-								this.fable.log.error(`Error getting server max UpdateDate ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
-								return fStageComplete(`Error getting server max UpdateDate ${this.EntitySchema.TableName}: ${pError}`);
+								this.fable.log.warn(`Could not get server max UpdateDate for ${this.EntitySchema.TableName} (${pError}); will sync by ID only.`);
+								return fStageComplete();
 							}
 							if (pBody && pBody.hasOwnProperty(this.DefaultIdentifier))
 							{
@@ -597,8 +629,9 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 						{
 							if (pError)
 							{
-								this.fable.log.error(`Error getting server count for ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
-								return fStageComplete(`Error getting server count for ${this.EntitySchema.TableName}: ${pError}`);
+								this.fable.log.warn(`Could not get server count for ${this.EntitySchema.TableName} (${pError}); estimating from max ID.`);
+								tmpSyncState.Server.RecordCount = tmpSyncState.Server.MaxIDEntity > 0 ? tmpSyncState.Server.MaxIDEntity : 0;
+								return fStageComplete();
 							}
 							if (pBody && pBody.hasOwnProperty('Count'))
 							{

package/source/services/clone/Meadow-Service-Sync.js

@@ -87,7 +87,7 @@ class MeadowSync extends libFableServiceProviderBase
 		let tmpErrorCount = 0;
 		let tmpSuccessCount = 0;
 
-		this.fable.Utility.eachLimit(this.MeadowSchemaTableList, 1,
+		this.fable.Utility.eachLimit(this.MeadowSchemaTableList, 5,
 			(pEntitySchemaName, fSyncInitializationComplete) =>
 			{
 				tmpEntityIndex++;