meadow-integration 1.0.9 → 1.0.10

package/examples/Example-011-Clone-With-SessionManager.js

@@ -0,0 +1,291 @@
+#!/usr/bin/env node
+/**
+ * Example 011: Clone with SessionManager
+ * ----------------------------------------
+ * Demonstrates using pict-sessionmanager to manage credentials for a
+ * data clone operation. The SessionManager handles authentication with
+ * the source API and automatically injects session credentials into
+ * all outbound REST requests during cloning.
+ *
+ * This example uses the programmatic API to set up SessionManager and
+ * run a clone, which is equivalent to what the CLI `data-clone` command
+ * does when SessionManager configuration is present in .meadow.config.json.
+ *
+ * Usage:  node Example-011-Clone-With-SessionManager.js
+ *
+ * Prerequisites:
+ *   - A running Meadow API server as the clone source
+ *   - A destination database (MySQL or MSSQL)
+ *   - A Meadow extended schema JSON file
+ *
+ * Configuration:
+ *   This example shows both inline configuration and file-based configuration.
+ *   In production, put your SessionManager config in .meadow.config.json.
+ */
+const libPict = require('pict');
+const libPath = require('path');
+
+const libMeadowConnectionManager = require('../source/services/clone/Meadow-Service-ConnectionManager.js');
+const libMeadowCloneRestClient = require('../source/services/clone/Meadow-Service-RestClient.js');
+const libMeadowSync = require('../source/services/clone/Meadow-Service-Sync.js');
+const libSessionManagerSetup = require('../source/Meadow-Integration-SessionManagerSetup.js');
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+// Source API configuration
+const tmpSourceConfig = {
+	ServerURL: 'https://myapp.example.com/1.0/',
+	// Note: UserID and Password are NOT set here because SessionManager
+	// handles authentication instead of the built-in mechanism.
+	UserID: false,
+	Password: false
+};
+
+// Destination database configuration
+const tmpDestinationConfig = {
+	Provider: 'MySQL',
+	MySQL: {
+		server: '127.0.0.1',
+		port: 3306,
+		user: 'root',
+		password: '',
+		database: 'meadow_clone',
+		connectionLimit: 20
+	}
+};
+
+// Sync configuration
+const tmpSyncConfig = {
+	DefaultSyncMode: 'Initial',
+	PageSize: 100,
+	SyncEntityList: [],
+	SyncEntityOptions: {}
+};
+
+// SessionManager configuration
+// This is the key addition -- it defines how to authenticate with the
+// source API and how credentials are injected into REST requests.
+const tmpSessionManagerConfig = {
+	Sessions: {
+		// Each key is a session name. You can have multiple sessions
+		// for different APIs or security contexts.
+		SourceAPI: {
+			// Type can be 'Header', 'Cookie', or 'Both'
+			Type: 'Cookie',
+
+			// Authentication configuration
+			AuthenticationMethod: 'post',
+			AuthenticationURITemplate: 'https://myapp.example.com/1.0/Authenticate',
+			AuthenticationRequestBody: {
+				// Templates use Pict's {~D:Record.Key~} syntax.
+				// Record refers to the Credentials object.
+				UserName: '{~D:Record.UserName~}',
+				Password: '{~D:Record.Password~}'
+			},
+			AuthenticationRetryCount: 2,
+			AuthenticationRetryDebounce: 500,
+
+			// Session check configuration (optional, for verifying session validity)
+			CheckSessionURITemplate: 'https://myapp.example.com/1.0/CheckSession',
+			CheckSessionLoginMarkerType: 'boolean',
+			CheckSessionLoginMarker: 'LoggedIn',
+
+			// Credential injection -- how the session token gets into requests
+			// DomainMatch: requests to URLs containing this string get credentials
+			DomainMatch: 'myapp.example.com',
+
+			// For cookie-based auth, specify the cookie name and where in
+			// the authentication response to find the value
+			CookieName: 'UserSession',
+			CookieValueAddress: 'SessionID',
+
+			// Credentials to use for authentication
+			// In .meadow.config.json these would be in the file.
+			// You can also pass them at authenticate() time.
+			Credentials: {
+				UserName: 'admin',
+				Password: 'my-secret-password'
+			}
+		}
+	}
+};
+
+// ============================================================================
+// Alternative: Header-based authentication
+// ============================================================================
+// If your API uses header-based tokens instead of cookies, configure like this:
+//
+// const tmpSessionManagerConfigHeader = {
+// 	Sessions: {
+// 		SourceAPI: {
+// 			Type: 'Header',
+// 			AuthenticationMethod: 'get',
+// 			AuthenticationURITemplate: 'https://myapp.example.com/1.0/Authenticate/{~D:Record.LoginID~}/{~D:Record.LoginPassword~}',
+// 			DomainMatch: 'myapp.example.com',
+// 			HeaderName: 'x-session-token',
+// 			HeaderValueTemplate: '{~D:Record.Token~}',
+// 			Credentials: {
+// 				LoginID: 'admin',
+// 				LoginPassword: 'my-secret-password'
+// 			}
+// 		}
+// 	}
+// };
+
+// ============================================================================
+// Equivalent .meadow.config.json
+// ============================================================================
+// When using the CLI, all of the above goes into .meadow.config.json:
+//
+// {
+// 	"Source": {
+// 		"ServerURL": "https://myapp.example.com/1.0/"
+// 	},
+// 	"Destination": {
+// 		"Provider": "MySQL",
+// 		"MySQL": {
+// 			"server": "127.0.0.1",
+// 			"port": 3306,
+// 			"user": "root",
+// 			"password": "",
+// 			"database": "meadow_clone",
+// 			"connectionLimit": 20
+// 		}
+// 	},
+// 	"SchemaPath": "./schema/Model-Extended.json",
+// 	"Sync": {
+// 		"DefaultSyncMode": "Initial",
+// 		"PageSize": 100,
+// 		"SyncEntityList": [],
+// 		"SyncEntityOptions": {}
+// 	},
+// 	"SessionManager": {
+// 		"Sessions": {
+// 			"SourceAPI": {
+// 				"Type": "Cookie",
+// 				"AuthenticationMethod": "post",
+// 				"AuthenticationURITemplate": "https://myapp.example.com/1.0/Authenticate",
+// 				"AuthenticationRequestBody": {
+// 					"UserName": "{~D:Record.UserName~}",
+// 					"Password": "{~D:Record.Password~}"
+// 				},
+// 				"CheckSessionURITemplate": "https://myapp.example.com/1.0/CheckSession",
+// 				"CheckSessionLoginMarker": "LoggedIn",
+// 				"DomainMatch": "myapp.example.com",
+// 				"CookieName": "UserSession",
+// 				"CookieValueAddress": "SessionID",
+// 				"Credentials": {
+// 					"UserName": "admin",
+// 					"Password": "my-secret-password"
+// 				}
+// 			}
+// 		}
+// 	}
+// }
+//
+// Then run:  mdwint clone --schema_path ./schema/Model-Extended.json
+
+// ============================================================================
+// Clone Execution
+// ============================================================================
+
+console.log('=== Example 011: Clone with SessionManager ===\n');
+
+// Create a Pict instance (needed for SessionManager's template engine)
+let _Fable = new libPict({ LogLevel: 3 });
+
+// Register clone services
+_Fable.serviceManager.addServiceType('MeadowCloneRestClient', libMeadowCloneRestClient);
+_Fable.serviceManager.instantiateServiceProvider('MeadowCloneRestClient', tmpSourceConfig);
+
+_Fable.serviceManager.addServiceType('MeadowConnectionManager', libMeadowConnectionManager);
+_Fable.serviceManager.instantiateServiceProvider('MeadowConnectionManager', tmpDestinationConfig);
+
+_Fable.serviceManager.addServiceType('MeadowSync', libMeadowSync);
+
+// Initialize SessionManager from config
+let tmpSessionManager = libSessionManagerSetup.initializeSessionManager(_Fable, tmpSessionManagerConfig);
+if (tmpSessionManager)
+{
+	console.log('SessionManager initialized with sessions:', Object.keys(tmpSessionManager.sessions).join(', '));
+
+	// Connect SessionManager to the clone RestClient's internal HTTP client.
+	// This means every REST request made during cloning will automatically
+	// have session credentials injected.
+	libSessionManagerSetup.connectSessionManagerToRestClient(_Fable, _Fable.MeadowCloneRestClient.restClient);
+	console.log('SessionManager connected to clone RestClient.\n');
+}
+else
+{
+	console.log('No SessionManager sessions configured.\n');
+}
+
+// Run the clone workflow
+_Fable.Utility.waterfall(
+	[
+		// Step 1: Authenticate SessionManager sessions
+		(fStageComplete) =>
+		{
+			if (!tmpSessionManager)
+			{
+				return fStageComplete();
+			}
+
+			console.log('Step 1: Authenticating SessionManager sessions...');
+			libSessionManagerSetup.authenticateSessions(_Fable,
+				(pError) =>
+				{
+					if (pError)
+					{
+						console.error(`  Authentication failed: ${pError.message}`);
+						console.error('  (This is expected if there is no actual server running.)');
+						return fStageComplete(pError);
+					}
+					console.log('  All sessions authenticated.\n');
+					return fStageComplete();
+				});
+		},
+
+		// Step 2: Connect to destination database
+		// (fStageComplete) =>
+		// {
+		// 	console.log('Step 2: Connecting to destination database...');
+		// 	_Fable.MeadowConnectionManager.connect(
+		// 		(pError, pConnectionPool) =>
+		// 		{
+		// 			if (pError)
+		// 			{
+		// 				console.error(`  Database connection failed: ${pError.message}`);
+		// 				return fStageComplete(pError);
+		// 			}
+		// 			console.log('  Connected.\n');
+		// 			return fStageComplete(null, pConnectionPool);
+		// 		});
+		// },
+
+		// Step 3: Load schema and sync
+		// (pConnectionPool, fStageComplete) =>
+		// {
+		// 	const tmpSchemaPath = libPath.resolve('./schema/Model-Extended.json');
+		// 	console.log(`Step 3: Loading schema from ${tmpSchemaPath}...`);
+		// 	// ... load schema and run sync
+		// },
+	],
+	(pError) =>
+	{
+		if (pError)
+		{
+			console.error(`\nClone ended with error: ${pError.message}`);
+			console.log('\nNote: This example is designed to demonstrate the configuration');
+			console.log('pattern. To run a real clone, point the Source and Destination');
+			console.log('settings at real servers and uncomment the database/sync steps.');
+		}
+		else
+		{
+			console.log('\nClone complete.');
+		}
+
+		console.log('\n=== Example Complete ===');
+	});

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "meadow-integration",
-    "version": "1.0.9",
+    "version": "1.0.10",
     "description": "Meadow Data Integration",
     "bin": {
         "mdwint": "source/cli/Meadow-Integration-CLI-Run.js"
@@ -40,10 +40,11 @@
         "fable": "^3.1.63",
         "fable-serviceproviderbase": "^3.0.19",
         "meadow": "^2.0.30",
-        "meadow-connection-mysql": "^1.0.14",
         "meadow-connection-mssql": "^1.0.16",
+        "meadow-connection-mysql": "^1.0.14",
         "orator": "^6.0.4",
         "orator-serviceserver-restify": "^2.0.9",
-        "pict-service-commandlineutility": "^1.0.19"
+        "pict-service-commandlineutility": "^1.0.19",
+        "pict-sessionmanager": "^1.0.2"
     }
 }

package/source/Meadow-Integration-SessionManagerSetup.js

@@ -0,0 +1,134 @@
+/**
+ * Meadow Integration - Session Manager Setup
+ *
+ * Utility to initialize pict-sessionmanager from configuration.
+ * Used by CLI, Console UI, and REST server entry points.
+ * Modules that use meadow-integration services directly should
+ * manage their own session lifecycle.
+ *
+ * @author Steven Velozo <steven@velozo.com>
+ */
+const libPictSessionManager = require('pict-sessionmanager');
+
+/**
+ * Initialize a SessionManager on a fable/pict instance from a config object.
+ *
+ * @param {object} pFable - A fable or pict instance
+ * @param {object} pSessionManagerConfig - The SessionManager configuration block
+ * @param {object} pSessionManagerConfig.Sessions - Map of session name to session configuration
+ * @returns {object|false} The instantiated SessionManager, or false if no sessions configured
+ */
+function initializeSessionManager(pFable, pSessionManagerConfig)
+{
+	if (!pFable)
+	{
+		return false;
+	}
+	if (!pSessionManagerConfig || typeof(pSessionManagerConfig) !== 'object')
+	{
+		return false;
+	}
+	if (!pSessionManagerConfig.Sessions || typeof(pSessionManagerConfig.Sessions) !== 'object')
+	{
+		return false;
+	}
+
+	let tmpSessionNames = Object.keys(pSessionManagerConfig.Sessions);
+	if (tmpSessionNames.length < 1)
+	{
+		return false;
+	}
+
+	pFable.serviceManager.addServiceType('SessionManager', libPictSessionManager);
+	pFable.serviceManager.instantiateServiceProvider('SessionManager');
+
+	for (let i = 0; i < tmpSessionNames.length; i++)
+	{
+		let tmpSessionName = tmpSessionNames[i];
+		let tmpSessionConfig = pSessionManagerConfig.Sessions[tmpSessionName];
+		pFable.SessionManager.addSession(tmpSessionName, tmpSessionConfig);
+	}
+
+	return pFable.SessionManager;
+}
+
+/**
+ * Authenticate all configured sessions that have Credentials set.
+ *
+ * @param {object} pFable - A fable or pict instance with SessionManager instantiated
+ * @param {function} fCallback - Callback (pError)
+ */
+function authenticateSessions(pFable, fCallback)
+{
+	if (!pFable || !pFable.SessionManager)
+	{
+		return fCallback();
+	}
+
+	let tmpSessionNames = Object.keys(pFable.SessionManager.sessions);
+	if (tmpSessionNames.length < 1)
+	{
+		return fCallback();
+	}
+
+	pFable.Utility.eachLimit(tmpSessionNames, 1,
+		(pSessionName, fSessionCallback) =>
+		{
+			let tmpSession = pFable.SessionManager.getSession(pSessionName);
+			let tmpCredentials = tmpSession.Configuration.Credentials;
+
+			if (!tmpCredentials || typeof(tmpCredentials) !== 'object' || Object.keys(tmpCredentials).length < 1)
+			{
+				pFable.log.info(`SessionManager setup: Session [${pSessionName}] has no credentials configured; skipping authentication.`);
+				return fSessionCallback();
+			}
+
+			pFable.log.info(`SessionManager setup: Authenticating session [${pSessionName}]...`);
+			pFable.SessionManager.authenticate(pSessionName, tmpCredentials,
+				(pError, pSessionState) =>
+				{
+					if (pError)
+					{
+						pFable.log.error(`SessionManager setup: Failed to authenticate session [${pSessionName}]: ${pError.message}`);
+						return fSessionCallback(pError);
+					}
+
+					pFable.log.info(`SessionManager setup: Session [${pSessionName}] authenticated successfully.`);
+					return fSessionCallback();
+				});
+		},
+		(pError) =>
+		{
+			return fCallback(pError);
+		});
+}
+
+/**
+ * Connect the SessionManager to a RestClient so credentials are injected automatically.
+ *
+ * If no RestClient is provided, SessionManager will use the default
+ * RestClient on the fable/pict instance (instantiating it if needed).
+ *
+ * @param {object} pFable - A fable or pict instance with SessionManager instantiated
+ * @param {object} [pRestClient] - A fable RestClient instance to connect to (optional)
+ * @returns {boolean} True if connected
+ */
+function connectSessionManagerToRestClient(pFable, pRestClient)
+{
+	if (!pFable || !pFable.SessionManager)
+	{
+		return false;
+	}
+
+	// SessionManager.connectToRestClient handles null by using the default
+	// pict RestClient (instantiating it if necessary).
+	pFable.SessionManager.connectToRestClient(pRestClient);
+	return true;
+}
+
+module.exports = (
+	{
+		initializeSessionManager: initializeSessionManager,
+		authenticateSessions: authenticateSessions,
+		connectSessionManagerToRestClient: connectSessionManagerToRestClient
+	});

package/source/cli/Default-Meadow-Integration-Configuration.json

@@ -34,5 +34,9 @@
 		"PageSize": 100,
 		"SyncEntityList": [],
 		"SyncEntityOptions": {}
+	},
+	"SessionManager":
+	{
+		"Sessions": {}
 	}
 }

package/source/cli/commands/Meadow-Integration-Command-DataClone.js

@@ -5,6 +5,7 @@ const libPath = require('path');
 const libMeadowConnectionManager = require('../../services/clone/Meadow-Service-ConnectionManager.js');
 const libMeadowCloneRestClient = require('../../services/clone/Meadow-Service-RestClient.js');
 const libMeadowSync = require('../../services/clone/Meadow-Service-Sync.js');
+const libSessionManagerSetup = require('../../Meadow-Integration-SessionManagerSetup.js');
 
 class DataClone extends libCLICommandLineCommand
 {
@@ -163,11 +164,40 @@ class DataClone extends libCLICommandLineCommand
 
 		this.fable.serviceManager.addServiceType('MeadowSync', libMeadowSync);
 
+		// Initialize SessionManager if configured
+		let tmpSessionManager = libSessionManagerSetup.initializeSessionManager(this.fable, tmpConfig.SessionManager);
+		if (tmpSessionManager)
+		{
+			// Connect SessionManager to the clone RestClient so credentials are auto-injected
+			libSessionManagerSetup.connectSessionManagerToRestClient(this.fable, this.fable.MeadowCloneRestClient.restClient);
+		}
+
 		this.fable.Utility.waterfall(
 			[
 				(fStageComplete) =>
 				{
-					// Authenticate with the source API
+					// Authenticate SessionManager sessions (if configured)
+					if (tmpSessionManager)
+					{
+						this.log.info('Authenticating SessionManager sessions...');
+						libSessionManagerSetup.authenticateSessions(this.fable,
+							(pError) =>
+							{
+								if (pError)
+								{
+									this.log.error('Error authenticating SessionManager sessions.', pError);
+								}
+								return fStageComplete();
+							});
+					}
+					else
+					{
+						return fStageComplete();
+					}
+				},
+				(fStageComplete) =>
+				{
+					// Authenticate with the source API using built-in credentials
 					if (tmpConfig.Source.UserID && tmpConfig.Source.Password)
 					{
 						this.log.info('Authenticating with source API...');
@@ -187,7 +217,7 @@ class DataClone extends libCLICommandLineCommand
 					}
 					else
 					{
-						this.log.info('No credentials configured; skipping authentication.');
+						this.log.info('No Source credentials configured; skipping built-in authentication.');
 						return fStageComplete();
 					}
 				},

package/source/cli/commands/Meadow-Integration-Command-Serve.js

@@ -30,10 +30,15 @@ class MeadowIntegrationCommandServe extends libCommandLineCommand
 
 		this.log.info(`Starting Meadow Integration REST server on port ${tmpPort}...`);
 
-		let tmpServer = new MeadowIntegrationServer(
-			{
-				APIServerPort: tmpPort
-			});
+		let tmpServerSettings = { APIServerPort: tmpPort };
+
+		// Pass SessionManager config from program configuration to the server
+		if (this.fable.ProgramConfiguration && this.fable.ProgramConfiguration.SessionManager)
+		{
+			tmpServerSettings.SessionManager = this.fable.ProgramConfiguration.SessionManager;
+		}
+
+		let tmpServer = new MeadowIntegrationServer(tmpServerSettings);
 
 		tmpServer.start(
 			(pError) =>

package/source/restserver/Meadow-Integration-Server.js

@@ -3,6 +3,7 @@ const libOrator = require('orator');
 const libOratorServiceServerRestify = require('orator-serviceserver-restify');
 
 const libEndpoints = require('./Meadow-Integration-Server-Endpoints.js');
+const libSessionManagerSetup = require('../Meadow-Integration-SessionManagerSetup.js');
 
 class MeadowIntegrationServer
 {
@@ -34,37 +35,63 @@ class MeadowIntegrationServer
 		this._Fable.instantiateServiceProvider('CSVParser');
 		this._Fable.instantiateServiceProvider('FilePersistence');
 		this._Fable.instantiateServiceProvider('DataGeneration');
+
+		// Initialize SessionManager if configured
+		this._SessionManager = libSessionManagerSetup.initializeSessionManager(this._Fable, tmpSettings.SessionManager);
 	}
 
 	start(fCallback)
 	{
 		let tmpCallback = (typeof(fCallback) === 'function') ? fCallback : () => {};
 
-		this._Orator.initialize(
-			(pError) =>
-			{
-				if (pError)
+		// Authenticate SessionManager sessions before starting the server
+		let fStartServer = () =>
+		{
+			this._Orator.initialize(
+				(pError) =>
 				{
-					this._Fable.log.error(`Error initializing Orator: ${pError}`, pError);
-					return tmpCallback(pError);
-				}
+					if (pError)
+					{
+						this._Fable.log.error(`Error initializing Orator: ${pError}`, pError);
+						return tmpCallback(pError);
+					}
 
-				// Register all endpoints
-				libEndpoints.connectRoutes(this._Fable, this._Orator);
+					// Register all endpoints
+					libEndpoints.connectRoutes(this._Fable, this._Orator);
 
-				this._Orator.startService(
-					(pStartError) =>
-					{
-						if (pStartError)
+					this._Orator.startService(
+						(pStartError) =>
 						{
-							this._Fable.log.error(`Error starting Orator service: ${pStartError}`, pStartError);
-							return tmpCallback(pStartError);
-						}
-
-						this._Fable.log.info(`Meadow Integration Server running on port ${this._Settings.APIServerPort}`);
-						return tmpCallback();
-					});
-			});
+							if (pStartError)
+							{
+								this._Fable.log.error(`Error starting Orator service: ${pStartError}`, pStartError);
+								return tmpCallback(pStartError);
+							}
+
+							this._Fable.log.info(`Meadow Integration Server running on port ${this._Settings.APIServerPort}`);
+							return tmpCallback();
+						});
+				});
+		};
+
+		if (this._SessionManager)
+		{
+			// Connect SessionManager to the default RestClient for outbound calls
+			libSessionManagerSetup.connectSessionManagerToRestClient(this._Fable);
+			libSessionManagerSetup.authenticateSessions(this._Fable,
+				(pAuthError) =>
+				{
+					if (pAuthError)
+					{
+						this._Fable.log.error(`Error authenticating SessionManager sessions: ${pAuthError.message}`);
+					}
+					fStartServer();
+				});
+		}
+		else
+		{
+			fStartServer();
+		}
 	}
 
 	stop(fCallback)

package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js

@@ -40,6 +40,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 		this.DefaultIdentifier = this.EntitySchema.MeadowSchema.DefaultIdentifier;
 		this.PageSize = this.options.PageSize || 100;
 		this.SyncDeletedRecords = this.options.SyncDeletedRecords || false;
+		this.MaxRecordsPerEntity = this.options.MaxRecordsPerEntity || 0;
 
 		this.Meadow = false;
 
@@ -197,8 +198,11 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 				this.fable.log.info(`Found ${tmpDeletedCount} deleted records on server for ${this.EntitySchema.TableName}; syncing deletions...`);
 
 				// Generate paginated URLs for deleted records
+				let tmpDeleteCap = (this.MaxRecordsPerEntity > 0)
+					? Math.min(tmpDeletedCount, this.MaxRecordsPerEntity)
+					: tmpDeletedCount;
 				const tmpDeleteURLPartials = [];
-				for (let i = 0; i < tmpDeletedCount; i += this.PageSize)
+				for (let i = 0; i < tmpDeleteCap; i += this.PageSize)
 				{
 					tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
 				}
@@ -307,6 +311,19 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 				Server: { MaxIDEntity: -1, RecordCount: 0 },
 			});
 
+		// Detect whether the table has a Deleted column
+		if (this.EntitySchema.MeadowSchema && Array.isArray(this.EntitySchema.MeadowSchema.Schema))
+		{
+			for (let i = 0; i < this.EntitySchema.MeadowSchema.Schema.length; i++)
+			{
+				const tmpColumn = this.EntitySchema.MeadowSchema.Schema[i];
+				if (tmpColumn.Type == 'Deleted' || tmpColumn.Column == 'Deleted')
+				{
+					tmpSyncState.HasDeletedColumn = true;
+				}
+			}
+		}
+
 		this.fable.Utility.waterfall(
 			[
 				(fStageComplete) =>
@@ -315,6 +332,10 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 					const tmpQuery = this.Meadow.query;
 					tmpQuery.setSort({ Column: this.DefaultIdentifier, Direction: 'Descending' });
 					tmpQuery.setCap(1);
+					if (!tmpSyncState.HasDeletedColumn)
+					{
+						tmpQuery.setDisableDeleteTracking(true);
+					}
 					this.Meadow.doRead(tmpQuery,
 						(pReadError, pQuery, pRecord) =>
 						{
@@ -333,6 +354,10 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 				{
 					// Get the count from local database
 					const tmpQuery = this.Meadow.query;
+					if (!tmpSyncState.HasDeletedColumn)
+					{
+						tmpQuery.setDisableDeleteTracking(true);
+					}
 					this.Meadow.doCount(tmpQuery,
 						(pCountError, pQuery, pCount) =>
 						{
@@ -382,17 +407,27 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 				{
 					tmpSyncState.EstimatedRecordCount = tmpSyncState.Server.RecordCount - tmpSyncState.Local.RecordCount;
 
+					// Apply MaxRecordsPerEntity cap if configured
+					let tmpRecordCap = (this.MaxRecordsPerEntity > 0)
+						? Math.min(tmpSyncState.Server.RecordCount, this.MaxRecordsPerEntity)
+						: tmpSyncState.Server.RecordCount;
+
+					if (this.MaxRecordsPerEntity > 0 && tmpSyncState.EstimatedRecordCount > this.MaxRecordsPerEntity)
+					{
+						tmpSyncState.EstimatedRecordCount = this.MaxRecordsPerEntity;
+					}
+
 					this.operation.createProgressTracker(tmpSyncState.EstimatedRecordCount, `FullSync-${this.EntitySchema.TableName}`);
 					this.operation.printProgressTrackerStatus(`FullSync-${this.EntitySchema.TableName}`);
 
 					// Generate paginated URL partials
 					tmpSyncState.URLPartials = [];
-					for (let i = 0; i < tmpSyncState.Server.RecordCount; i += this.PageSize)
+					for (let i = 0; i < tmpRecordCap; i += this.PageSize)
 					{
 						tmpSyncState.URLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~${this.DefaultIdentifier}~GT~${tmpSyncState.Local.MaxIDEntity}~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
 					}
 
-					this.fable.log.info(`${this.EntitySchema.TableName}: downloading ${tmpSyncState.URLPartials.length} pages (local: ${tmpSyncState.Local.RecordCount}/${tmpSyncState.Local.MaxIDEntity}, server: ${tmpSyncState.Server.RecordCount}/${tmpSyncState.Server.MaxIDEntity}, estimated new: ${tmpSyncState.EstimatedRecordCount})`);
+					this.fable.log.info(`${this.EntitySchema.TableName}: downloading ${tmpSyncState.URLPartials.length} pages (local: ${tmpSyncState.Local.RecordCount}/${tmpSyncState.Local.MaxIDEntity}, server: ${tmpSyncState.Server.RecordCount}/${tmpSyncState.Server.MaxIDEntity}, estimated new: ${tmpSyncState.EstimatedRecordCount}${this.MaxRecordsPerEntity > 0 ? `, capped at ${this.MaxRecordsPerEntity}` : ''})`);
 
 					return fStageComplete();
 				},
@@ -429,6 +464,11 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 													tmpQuery.addFilter(this.DefaultIdentifier, tmpRecord[this.DefaultIdentifier]);
 												}
 
+												if (!tmpSyncState.HasDeletedColumn)
+												{
+													tmpQuery.setDisableDeleteTracking(true);
+												}
+
 												this.Meadow.doRead(tmpQuery,
 													(pReadError, pQuery, pRecord) =>
 													{
@@ -456,6 +496,31 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
 																{
 																	if (pCreateError)
 																	{
+																		let tmpErrorStr = (typeof(pCreateError) === 'string') ? pCreateError : JSON.stringify(pCreateError);
+																		if (tmpErrorStr.toLowerCase().indexOf('duplicate') > -1 || tmpErrorStr.toLowerCase().indexOf('unique') > -1)
+																		{
+																			// Duplicate key (likely GUID conflict) -- fall back to update
+																			this.log.warn(`${this.EntitySchema.TableName}: duplicate key on create for ID ${tmpRecord[this.DefaultIdentifier]}; falling back to update.`);
+																			const tmpUpdateQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+																			tmpUpdateQuery.setDisableAutoIdentity(true);
+																			tmpUpdateQuery.setDisableAutoDateStamp(true);
+																			tmpUpdateQuery.setDisableAutoUserStamp(true);
+																			tmpUpdateQuery.setDisableDeleteTracking(true);
+																			this.Meadow.doUpdate(tmpUpdateQuery,
+																				(pUpdateError) =>
+																				{
+																					if (pUpdateError)
+																					{
+																						tmpRecordsErrored++;
+																						this.log.error(`${this.EntitySchema.TableName}: fallback update also failed for ID ${tmpRecord[this.DefaultIdentifier]}: ${pUpdateError}`);
+																						return fEntitySyncComplete();
+																					}
+																					tmpRecordsCreated++;
+																					this.operation.incrementProgressTrackerStatus(`FullSync-${this.EntitySchema.TableName}`, 1);
+																					return fEntitySyncComplete();
+																				});
+																			return;
+																		}
 																		tmpRecordsErrored++;
 																		this.log.error(`${this.EntitySchema.TableName}: doCreate error for ID ${tmpRecord[this.DefaultIdentifier]}: ${pCreateError}`);
 																		return fEntitySyncComplete();

package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js

@@ -40,6 +40,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 		this.DefaultIdentifier = this.EntitySchema.MeadowSchema.DefaultIdentifier;
 		this.PageSize = this.options.PageSize || 100;
 		this.SyncDeletedRecords = this.options.SyncDeletedRecords || false;
+		this.MaxRecordsPerEntity = this.options.MaxRecordsPerEntity || 0;
 
 		this.Meadow = false;
 
@@ -117,7 +118,14 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 						}
 						break;
 					default:
-						if (pSourceRecord[tmpColumn.Column] !== '')
+						if (tmpColumn.DataType == 'DateTime')
+						{
+							if ((typeof(pSourceRecord[tmpColumn.Column]) == 'string') && (pSourceRecord[tmpColumn.Column].length > 0))
+							{
+								tmpRecordToCommit[tmpColumn.Column] = this.fable.Dates.dayJS.utc(pSourceRecord[tmpColumn.Column]).format('YYYY-MM-DD HH:mm:ss.SSS');
+							}
+						}
+						else if (pSourceRecord[tmpColumn.Column] !== '')
 						{
 							tmpRecordToCommit[tmpColumn.Column] = pSourceRecord[tmpColumn.Column];
 						}
@@ -165,8 +173,11 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 				this.fable.log.info(`Found ${tmpDeletedCount} deleted records on server for ${this.EntitySchema.TableName}; syncing deletions...`);
 
 				// Generate paginated URLs for deleted records
+				let tmpDeleteCap = (this.MaxRecordsPerEntity > 0)
+					? Math.min(tmpDeletedCount, this.MaxRecordsPerEntity)
+					: tmpDeletedCount;
 				const tmpDeleteURLPartials = [];
-				for (let i = 0; i < tmpDeletedCount; i += this.PageSize)
+				for (let i = 0; i < tmpDeleteCap; i += this.PageSize)
 				{
 					tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
 				}
@@ -298,6 +309,11 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 											tmpQuery.addFilter(this.DefaultIdentifier, tmpRecord[this.DefaultIdentifier]);
 										}
 
+										if (!tmpSyncState.HasDeletedColumn)
+										{
+											tmpQuery.setDisableDeleteTracking(true);
+										}
+
 										this.Meadow.doRead(tmpQuery,
 											(pReadError, pQuery, pRecord) =>
 											{
@@ -338,6 +354,29 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 														{
 															if (pCreateError)
 															{
+																let tmpErrorStr = (typeof(pCreateError) === 'string') ? pCreateError : JSON.stringify(pCreateError);
+																if (tmpErrorStr.toLowerCase().indexOf('duplicate') > -1 || tmpErrorStr.toLowerCase().indexOf('unique') > -1)
+																{
+																	// Duplicate key (likely GUID conflict) -- fall back to update
+																	this.log.warn(`Duplicate key on create for ${this.EntitySchema.TableName} ID ${tmpRecord[this.DefaultIdentifier]}; falling back to update.`);
+																	const tmpFallbackQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
+																	tmpFallbackQuery.setDisableAutoIdentity(true);
+																	tmpFallbackQuery.setDisableAutoDateStamp(true);
+																	tmpFallbackQuery.setDisableAutoUserStamp(true);
+																	tmpFallbackQuery.setDisableDeleteTracking(true);
+																	this.Meadow.doUpdate(tmpFallbackQuery,
+																		(pUpdateError) =>
+																		{
+																			if (pUpdateError)
+																			{
+																				this.log.error(`Fallback update also failed for ${this.EntitySchema.TableName} ID ${tmpRecord[this.DefaultIdentifier]}: ${pUpdateError}`);
+																				return fNextEntityRecordSync();
+																			}
+																			this.operation.incrementProgressTrackerStatus(`UpdateSync-${this.EntitySchema.TableName}`, 1);
+																			return fNextEntityRecordSync();
+																		});
+																	return;
+																}
 																this.log.error(`Error creating record ${this.EntitySchema.TableName}: ${pCreateError}`, pCreateError);
 																return fNextEntityRecordSync();
 															}
@@ -407,11 +446,18 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 						{
 							tmpSyncState.Local.HasUpdateDate = true;
 							tmpSyncState.Server.HasUpdateDate = true;
-							this.log.info(`Entity ${this.EntitySchema.TableName} has UpdateDate column.`);
-							break;
+						}
+						if (tmpColumn.Type == 'Deleted' || tmpColumn.Column == 'Deleted')
+						{
+							tmpSyncState.HasDeletedColumn = true;
 						}
 					}
 
+					if (tmpSyncState.Local.HasUpdateDate)
+					{
+						this.log.info(`Entity ${this.EntitySchema.TableName} has UpdateDate column.`);
+					}
+
 					this.log.info(`Syncing with UPDATE STRATEGY entity ${this.EntitySchema.TableName}...`);
 					return fStageComplete();
 				},
@@ -421,6 +467,11 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 					const tmpQuery = this.Meadow.query;
 					tmpQuery.setSort({ Column: this.DefaultIdentifier, Direction: 'Descending' });
 					tmpQuery.setCap(1);
+					// Disable delete tracking if the table has no Deleted column
+					if (!tmpSyncState.HasDeletedColumn)
+					{
+						tmpQuery.setDisableDeleteTracking(true);
+					}
 					this.Meadow.doRead(tmpQuery,
 						(pReadError, pQuery, pRecord) =>
 						{
@@ -441,10 +492,19 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 				},
 				(fStageComplete) =>
 				{
-					// Get the Max UpdateDate from local database
+					// Get the Max UpdateDate from local database — skip if table has no UpdateDate column
+					if (!tmpSyncState.Local.HasUpdateDate)
+					{
+						this.fable.log.info(`No UpdateDate column for ${this.EntitySchema.TableName}; skipping UpdateDate check.`);
+						return fStageComplete();
+					}
 					const tmpQuery = this.Meadow.query;
 					tmpQuery.setSort({ Column: 'UpdateDate', Direction: 'Descending' });
 					tmpQuery.setCap(1);
+					if (!tmpSyncState.HasDeletedColumn)
+					{
+						tmpQuery.setDisableDeleteTracking(true);
+					}
 					this.Meadow.doRead(tmpQuery,
 						(pReadError, pQuery, pRecord) =>
 						{
@@ -467,6 +527,10 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 				{
 					// Get the count from local database
 					const tmpQuery = this.Meadow.query;
+					if (!tmpSyncState.HasDeletedColumn)
+					{
+						tmpQuery.setDisableDeleteTracking(true);
+					}
 					this.Meadow.doCount(tmpQuery,
 						(pCountError, pQuery, pCount) =>
 						{
@@ -550,7 +614,10 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
 				},
 				(fStageComplete) =>
 				{
-					tmpSyncState.EstimatedRequestCount = Math.ceil(tmpSyncState.Server.RecordCount / this.PageSize);
+					let tmpTotalRecords = (this.MaxRecordsPerEntity > 0)
+						? Math.min(tmpSyncState.Server.RecordCount, this.MaxRecordsPerEntity)
+						: tmpSyncState.Server.RecordCount;
+					tmpSyncState.EstimatedRequestCount = Math.ceil(tmpTotalRecords / this.PageSize);
 					tmpSyncState.RequestsPerformed = 0;
 					tmpSyncState.LastRequestedID = 0;
 

package/source/services/clone/Meadow-Service-Sync.js

@@ -54,6 +54,17 @@ class MeadowSync extends libFableServiceProviderBase
 			this.SyncDeletedRecords = !!this.options.SyncDeletedRecords;
 		}
 
+		// When > 0, limit sync to at most this many records per entity.
+		this.MaxRecordsPerEntity = 0;
+		if (this.fable.ProgramConfiguration.hasOwnProperty('MaxRecordsPerEntity'))
+		{
+			this.MaxRecordsPerEntity = parseInt(this.fable.ProgramConfiguration.MaxRecordsPerEntity, 10) || 0;
+		}
+		else if (this.options.hasOwnProperty('MaxRecordsPerEntity'))
+		{
+			this.MaxRecordsPerEntity = parseInt(this.options.MaxRecordsPerEntity, 10) || 0;
+		}
+
 		this.MeadowSchema = false;
 		this.MeadowSchemaTableList = false;
 
@@ -89,6 +100,7 @@ class MeadowSync extends libFableServiceProviderBase
 						ConnectionPool: this.options.ConnectionPool,
 						PageSize: this.options.PageSize || 100,
 						SyncDeletedRecords: this.SyncDeletedRecords,
+						MaxRecordsPerEntity: this.MaxRecordsPerEntity,
 					};
 
 					let tmpSyncEntity;