meadow-integration 1.0.17 → 1.0.19

package/source/Meadow-Service-Integration-GUIDMap.js

@@ -65,7 +65,16 @@ class MeadowGUIDMap extends libFableServiceProviderBase
 		return (this._GUIDMap[pEntity].hasOwnProperty(pGUID)) ? this._GUIDMap[pEntity][pGUID] : false;
 	}
 
-	getIDFromGUIDAsync(pEntity, pGUID, fCallback)
+	/**
+	 * Get an ID from a GUID, loading from the server if not already cached.
+	 *
+	 * @param {string} pEntity - The entity name
+	 * @param {string} pGUID - The GUID to look up
+	 * @param {function} fCallback - Callback (pError, pID)
+	 * @param {object} [pClient] - Optional REST client to use for the server lookup
+	 *                              (defaults to this.fable.MeadowRestClient or this.fable.MeadowCloneRestClient)
+	 */
+	getIDFromGUIDAsync(pEntity, pGUID, fCallback, pClient)
 	{
 		if (!this._GUIDMap.hasOwnProperty(pEntity))
 		{
@@ -79,8 +88,16 @@ class MeadowGUIDMap extends libFableServiceProviderBase
 		}
 		else
 		{
+			// Resolve the REST client to use for the server lookup
+			let tmpClient = pClient || this.fable.MeadowRestClient || this.fable.MeadowCloneRestClient;
+
+			if (!tmpClient || typeof(tmpClient.getEntityByGUID) !== 'function')
+			{
+				return fCallback(new Error(`No REST client with getEntityByGUID available for GUID lookup of [${pEntity}].[${pGUID}].`), false);
+			}
+
 			// Try to load it from the server
-			this.fable.MeadowRestClient.getEntityByGUID(pEntity, pGUID,
+			tmpClient.getEntityByGUID(pEntity, pGUID,
 				(pError, pBody) =>
 				{
 					if (pError)

package/source/cli/commands/Meadow-Integration-Command-ComprehensionPush.js

@@ -3,6 +3,8 @@ const libCLICommandLineCommand = require('pict-service-commandlineutility').Serv
 const libPath = require('path');
 
 const libIntegrationAdapter = require('../../Meadow-Service-Integration-Adapter.js');
+const libMeadowCloneRestClient = require('../../services/clone/Meadow-Service-RestClient.js');
+const libSessionManagerSetup = require('../../Meadow-Integration-SessionManagerSetup.js');
 
 class PushComprehensionsViaIntegration extends libCLICommandLineCommand
 {
@@ -15,17 +17,52 @@ class PushComprehensionsViaIntegration extends libCLICommandLineCommand
 		this.options.Aliases.push('load');
 		this.options.Aliases.push('push');
 
-
 		this.options.CommandArguments.push({ Name: '<comprehension_file>', Description: 'The comprehension file path.' });
 
-		this.options.CommandOptions.push({ Name: '-p, --prefix [guid_prefix]', Description: 'GUID Prefix for the comprehension push.'});
-		this.options.CommandOptions.push({ Name: '-e, --entityguidprefix [entity_guid_prefix]', Description: 'GUID Prefix for each entity.'});
+		this.options.CommandOptions.push({ Name: '-p, --prefix [guid_prefix]', Description: 'GUID Prefix for the comprehension push.' });
+		this.options.CommandOptions.push({ Name: '-e, --entityguidprefix [entity_guid_prefix]', Description: 'GUID Prefix for each entity.' });
+
+		this.options.CommandOptions.push({ Name: '-a, --api_server [api_server]', Description: 'The API server URL.' });
+		this.options.CommandOptions.push({ Name: '-u, --api_username [api_username]', Description: 'The API username to authenticate with.' });
+		this.options.CommandOptions.push({ Name: '-w, --api_password [api_password]', Description: 'The API password to authenticate with.' });
+
+		this.options.CommandOptions.push({ Name: '--bulkupsert [bulk_upsert]', Description: 'Enable bulk upsert mode (true/false). Default: true.', DefaultValue: 'true' });
+		this.options.CommandOptions.push({ Name: '--batchsize [batch_size]', Description: 'Bulk upsert batch size. Default: 100.', DefaultValue: '100' });
+		this.options.CommandOptions.push({ Name: '--progressinterval [progress_interval]', Description: 'Per-entity progress log interval (records). Default: 100.', DefaultValue: '100' });
+		this.options.CommandOptions.push({ Name: '--metaprogressinterval [meta_progress_interval]', Description: 'Meta (cross-entity) progress log interval. Default: 0 (disabled).', DefaultValue: '0' });
+		this.options.CommandOptions.push({ Name: '--allowguidtruncation', Description: 'Allow automatic GUID prefix truncation when GUIDs exceed max length.' });
+		this.options.CommandOptions.push({ Name: '--logfile [logfile_path]', Description: 'Path to write log output.' });
 
 		this.addCommand();
 
 		this.comprehension = {};
 	}
 
+	_resolveConfig()
+	{
+		const tmpConfig = JSON.parse(JSON.stringify(this.fable.ProgramConfiguration));
+
+		// Apply command-line overrides for Source (API)
+		if (!tmpConfig.Source)
+		{
+			tmpConfig.Source = {};
+		}
+		if (this.CommandOptions.api_server)
+		{
+			tmpConfig.Source.ServerURL = this.CommandOptions.api_server;
+		}
+		if (this.CommandOptions.api_username)
+		{
+			tmpConfig.Source.UserID = this.CommandOptions.api_username;
+		}
+		if (this.CommandOptions.api_password)
+		{
+			tmpConfig.Source.Password = this.CommandOptions.api_password;
+		}
+
+		return tmpConfig;
+	}
+
 	runAdapter(pAnticipate, pAdapter, pDataMap, fMarshalRecord)
 	{
 		let tmpAdapter = this.fable.servicesMap.IntegrationAdapter[pAdapter];
@@ -76,56 +113,191 @@ class PushComprehensionsViaIntegration extends libCLICommandLineCommand
 	pushComprehension(fCallback)
 	{
 		let tmpComprehensionPath = this.ArgumentString;
+		let tmpConfig = this._resolveConfig();
 
-		let tmpAnticipate = this.fable.newAnticipate();
 		this.fable.log.info(`Pushing comprehension file [${tmpComprehensionPath}] to the Meadow Endpoints APIs.`);
 
+		// --- Setup REST client ---
+		this.fable.serviceManager.addServiceType('MeadowCloneRestClient', libMeadowCloneRestClient);
+
+		let tmpRestClientOptions = {};
+		if (tmpConfig.Source && tmpConfig.Source.ServerURL)
+		{
+			tmpRestClientOptions.ServerURL = tmpConfig.Source.ServerURL;
+		}
+		if (tmpConfig.Source && tmpConfig.Source.UserID)
+		{
+			tmpRestClientOptions.UserID = tmpConfig.Source.UserID;
+		}
+		if (tmpConfig.Source && tmpConfig.Source.Password)
+		{
+			tmpRestClientOptions.Password = tmpConfig.Source.Password;
+		}
+
+		this.fable.serviceManager.instantiateServiceProvider('MeadowCloneRestClient', tmpRestClientOptions);
+
+		// --- Initialize SessionManager if configured ---
+		let tmpSessionManager = libSessionManagerSetup.initializeSessionManager(this.fable, tmpConfig.SessionManager);
+		if (tmpSessionManager)
+		{
+			libSessionManagerSetup.connectSessionManagerToRestClient(this.fable, this.fable.MeadowCloneRestClient.restClient);
+		}
+
+		// --- Setup Integration Adapter service type ---
 		this.fable.log.info(`Initializing and configuring data integration adapters...`);
 		this.fable.serviceManager.addServiceType('IntegrationAdapter', libIntegrationAdapter);
 
-		tmpAnticipate.anticipate(
-			function (fCallback)
-			{
-				try
+		// --- Resolve adapter options from CLI flags + config ---
+		let tmpBulkUpsertEnabled = (this.CommandOptions.bulkupsert !== 'false');
+		let tmpBatchSize = parseInt(this.CommandOptions.batchsize) || 100;
+		let tmpProgressInterval = parseInt(this.CommandOptions.progressinterval) || 100;
+		let tmpMetaProgressInterval = parseInt(this.CommandOptions.metaprogressinterval) || 0;
+		let tmpAllowGUIDTruncation = !!this.CommandOptions.allowguidtruncation;
+
+		// Build shared adapter options
+		let tmpAdapterOptions = {
+			SimpleMarshal: true,
+			ForceMarshal: true,
+			BulkUpsertBatchSize: tmpBatchSize,
+			RecordThresholdForBulkUpsert: tmpBulkUpsertEnabled ? 1000 : Infinity,
+			ProgressLogInterval: tmpProgressInterval,
+			AllowGUIDTruncation: tmpAllowGUIDTruncation
+		};
+
+		this.fable.Utility.waterfall(
+			[
+				(fStageComplete) =>
 				{
-					this.fable.log.info(`Loading Comprehension File...`);
-					tmpComprehensionPath = libPath.resolve(tmpComprehensionPath);
-					this.comprehension = require(tmpComprehensionPath);
-					return fCallback();
-				}
-				catch(pError)
+					// Authenticate SessionManager sessions (if configured)
+					if (tmpSessionManager)
+					{
+						this.log.info('Authenticating SessionManager sessions...');
+						libSessionManagerSetup.authenticateSessions(this.fable,
+							(pError) =>
+							{
+								if (pError)
+								{
+									this.log.error('Error authenticating SessionManager sessions.', pError);
+								}
+								return fStageComplete();
+							});
+					}
+					else
+					{
+						return fStageComplete();
+					}
+				},
+				(fStageComplete) =>
 				{
-					this.fable.log.error(`Error loading comprehension file [${tmpComprehensionPath}]: ${pError}`, pError);
-					return fCallback(pError);
-				}
-			}.bind(this));
-
-		tmpAnticipate.anticipate(
-			(fCallback) =>
-			{
-				this.fable.log.info(`Wiring up Integration Adapters...`);
+					// Authenticate with the API server using built-in credentials
+					if (tmpConfig.Source && tmpConfig.Source.UserID && tmpConfig.Source.Password)
+					{
+						this.log.info('Authenticating with API server...');
+						this.fable.MeadowCloneRestClient.authenticate(
+							(pError, pResponse) =>
+							{
+								if (pError)
+								{
+									this.log.error('Error authenticating with API server.', pError);
+								}
+								else
+								{
+									this.log.info(`Authenticated with API server as [${tmpConfig.Source.UserID}] at [${tmpConfig.Source.ServerURL}]`);
+								}
+								return fStageComplete();
+							});
+					}
+					else
+					{
+						this.log.info('No Source credentials configured; skipping built-in authentication.');
+						return fStageComplete();
+					}
+				},
+				(fStageComplete) =>
+				{
+					// Load comprehension file
+					try
+					{
+						this.fable.log.info(`Loading Comprehension File...`);
+						tmpComprehensionPath = libPath.resolve(tmpComprehensionPath);
+						this.comprehension = require(tmpComprehensionPath);
+						return fStageComplete();
+					}
+					catch (pError)
+					{
+						this.fable.log.error(`Error loading comprehension file [${tmpComprehensionPath}]: ${pError}`, pError);
+						return fStageComplete(pError);
+					}
+				},
+				(fStageComplete) =>
+				{
+					this.fable.log.info(`Wiring up Integration Adapters...`);
 
-				let tmpIntegrationAdapterSet = Object.keys(this.comprehension);
+					let tmpIntegrationAdapterSet = Object.keys(this.comprehension);
 
-				try
-				{
+					// Count total records for meta progress tracking
+					let tmpTotalRecords = 0;
 					for (let i = 0; i < tmpIntegrationAdapterSet.length; i++)
 					{
 						let tmpAdapterKey = tmpIntegrationAdapterSet[i];
-						libIntegrationAdapter.getAdapter(this.fable, tmpAdapterKey, this.getCapitalLettersAsString(tmpAdapterKey), { SimpleMarshal: true, ForceMarshal: true });
-						this.runAdapter(tmpAnticipate, tmpAdapterKey, this.comprehension[tmpAdapterKey]);
+						if (this.comprehension[tmpAdapterKey] && typeof(this.comprehension[tmpAdapterKey]) === 'object')
+						{
+							tmpTotalRecords += Object.keys(this.comprehension[tmpAdapterKey]).length;
+						}
 					}
-				}
-				catch (pError)
-				{
-					this.fable.log.error(`Error wiring up integration adapters: ${pError}`, pError);
-					return fCallback(pError);
-				}
 
-				return fCallback();
-			});
+					// Start meta progress tracker if interval is configured
+					let tmpMetaProgressHash = false;
+					if (tmpMetaProgressInterval > 0 && tmpTotalRecords > 0)
+					{
+						this.fable.instantiateServiceProviderIfNotExists('ProgressTrackerSet');
+						tmpMetaProgressHash = this.fable.getUUID();
+						this.fable.ProgressTrackerSet.createProgressTracker(tmpMetaProgressHash, tmpTotalRecords);
+						this.fable.ProgressTrackerSet.startProgressTracker(tmpMetaProgressHash);
+						this.fable.log.info(`Meta progress: ${tmpTotalRecords} total records across ${tmpIntegrationAdapterSet.length} entities.`);
+					}
 
-		tmpAnticipate.wait(
+					let tmpAnticipate = this.fable.newAnticipate();
+
+					try
+					{
+						for (let i = 0; i < tmpIntegrationAdapterSet.length; i++)
+						{
+							let tmpAdapterKey = tmpIntegrationAdapterSet[i];
+							let tmpAdapter = libIntegrationAdapter.getAdapter(this.fable, tmpAdapterKey, this.getCapitalLettersAsString(tmpAdapterKey), tmpAdapterOptions);
+
+							// Inject the REST client
+							tmpAdapter.setRestClient(this.fable.MeadowCloneRestClient);
+
+							// Wire up meta progress tracking
+							if (tmpMetaProgressHash)
+							{
+								tmpAdapter.MetaProgressTrackerHash = tmpMetaProgressHash;
+								tmpAdapter.MetaProgressTrackerLogInterval = tmpMetaProgressInterval;
+							}
+
+							this.runAdapter(tmpAnticipate, tmpAdapterKey, this.comprehension[tmpAdapterKey]);
+						}
+					}
+					catch (pError)
+					{
+						this.fable.log.error(`Error wiring up integration adapters: ${pError}`, pError);
+						return fStageComplete(pError);
+					}
+
+					tmpAnticipate.wait(
+						(pError) =>
+						{
+							// End meta progress tracker
+							if (tmpMetaProgressHash)
+							{
+								this.fable.ProgressTrackerSet.endProgressTracker(tmpMetaProgressHash);
+								this.fable.ProgressTrackerSet.logProgressTrackerStatus(tmpMetaProgressHash);
+							}
+							return fStageComplete(pError);
+						});
+				}
+			],
 			(pError) =>
 			{
 				if (pError)
@@ -140,7 +312,7 @@ class PushComprehensionsViaIntegration extends libCLICommandLineCommand
 
 	onRunAsync(fCallback)
 	{
-		return this.pushComprehension((pError)=> 
+		return this.pushComprehension((pError) =>
 		{
 			return fCallback(pError);
 		});

package/source/services/clone/Meadow-Service-RestClient.js

@@ -155,6 +155,52 @@ class MeadowCloneRestClient extends libFableServiceProviderBase
 			});
 	}
 
+	upsertEntities(pEntity, pRecordArray, fCallback)
+	{
+		let tmpRequestOptions = (
+			{
+				url: `${this.serverURL}${pEntity}s/Upserts`,
+				body: pRecordArray,
+			});
+		tmpRequestOptions = this._prepareRequestOptions(tmpRequestOptions);
+
+		this.restClient.putJSON(tmpRequestOptions,
+			(pError, pResponse, pBody) =>
+			{
+				if (pError)
+				{
+					this.log.error(`Error bulk upserting ${pEntity} records: ${pError.message}`);
+				}
+				return fCallback(pError, pBody);
+			});
+	}
+
+	getEntityByGUID(pEntity, pGUID, fCallback)
+	{
+		// Meadow's "ReadBy" endpoint uses the plural entity name with pagination:
+		//   GET /{Entity}s/By/GUID{Entity}/{Value}/{Start}/{Cap}
+		let tmpRequestOptions = (
+			{
+				url: `${this.serverURL}${pEntity}s/By/GUID${pEntity}/${pGUID}/0/1`,
+			});
+		tmpRequestOptions = this._prepareRequestOptions(tmpRequestOptions);
+
+		return this.restClient.getJSON(tmpRequestOptions,
+			(pError, pResponse, pBody) =>
+			{
+				if (pError)
+				{
+					this.log.error(`Error getting ${pEntity} by GUID [${pGUID}]: ${pError.message}`);
+				}
+				// The /By/ endpoint returns an array; extract the first record for convenience.
+				if (Array.isArray(pBody) && pBody.length > 0)
+				{
+					return fCallback(pError, pBody[0]);
+				}
+				return fCallback(pError, pBody);
+			});
+	}
+
 	deleteEntity(pEntity, pIDRecord, fCallback)
 	{
 		let tmpRequestOptions = (

package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js

@@ -448,9 +448,10 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 					tmpSyncState.EstimatedRecordCount = tmpSyncState.Server.RecordCount - tmpSyncState.Local.RecordCount;
 
 					// Apply MaxRecordsPerEntity cap if configured
-					let tmpRecordCap = (this.MaxRecordsPerEntity > 0)
+					tmpSyncState.RecordCap = (this.MaxRecordsPerEntity > 0)
 						? Math.min(tmpSyncState.Server.RecordCount, this.MaxRecordsPerEntity)
 						: tmpSyncState.Server.RecordCount;
+					let tmpRecordCap = tmpSyncState.RecordCap;
 
 					if (this.MaxRecordsPerEntity > 0 && tmpSyncState.EstimatedRecordCount > this.MaxRecordsPerEntity)
 					{
@@ -617,7 +618,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 
 						const fFetchPage = () =>
 						{
-							if (tmpTotalFetched >= tmpRecordCap)
+							if (tmpTotalFetched >= tmpSyncState.RecordCap)
 							{
 								return fSyncComplete();
 							}