meadow-connection-mssql 1.0.17 → 1.0.18

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "meadow-connection-mssql",
-    "version": "1.0.17",
+    "version": "1.0.18",
     "description": "Meadow MSSQL Plugin",
     "main": "source/Meadow-Connection-MSSQL.js",
     "scripts": {
@@ -48,7 +48,7 @@
     },
     "homepage": "https://github.com/stevenvelozo/meadow-connection-mssql",
     "devDependencies": {
-        "fable": "^3.1.63",
+        "fable": "^3.1.70",
         "pict-docuserve": "^0.1.5",
         "quackage": "^1.1.0"
     },

package/source/Meadow-Connection-MSSQL.js

@@ -7,6 +7,17 @@ const libFableServiceProviderBase = require('fable-serviceproviderbase');
 const libMSSQL = require('mssql');
 
 const libMeadowSchemaMSSQL = require('./Meadow-Schema-MSSQL.js');
+const libRetry = require('./Meadow-MSSQL-Retry.js');
+
+// Default timeouts and retry behavior.  All configurable per-provider via
+// the MSSQL options block in fable settings.  Defaults lean generous for
+// slow WAN links / firewalled customer networks — better to wait a minute
+// than to false-fail a real sync.
+const DEFAULT_REQUEST_TIMEOUT_MS    = 120000;  // 2 min per query
+const DEFAULT_CONNECTION_TIMEOUT_MS = 60000;   // 1 min to establish a connection
+const DEFAULT_CONNECT_MAX_ATTEMPTS  = 5;
+const DEFAULT_CONNECT_INITIAL_DELAY = 3000;
+const DEFAULT_CONNECT_MAX_DELAY     = 30000;
 
 /*
 	Das alt muster:
@@ -67,7 +78,15 @@ class MeadowConnectionMSSQL extends libFableServiceProviderBase
 					user: this.options.user,
 					password: this.options.password,
 					database: this.options.database,
-					ConnectionPoolLimit: this.options.ConnectionPoolLimit
+					ConnectionPoolLimit: this.options.ConnectionPoolLimit,
+					// Reliability tuning — forward through so it ends up on
+					// options.MSSQL where _buildConnectionSettings and
+					// _connectRetryOptions look for it.
+					RequestTimeoutMs: this.options.RequestTimeoutMs,
+					ConnectionTimeoutMs: this.options.ConnectionTimeoutMs,
+					ConnectRetryOptions: this.options.ConnectRetryOptions,
+					DDLRetryOptions: this.options.DDLRetryOptions,
+					LegacyPagination: this.options.LegacyPagination
 				});
 		}
 		else if (typeof(this.fable.settings.MSSQL) == 'object')
@@ -81,12 +100,20 @@ class MeadowConnectionMSSQL extends libFableServiceProviderBase
 					user: tmpSettings.user || tmpSettings.User,
 					password: tmpSettings.password || tmpSettings.Password,
 					database: tmpSettings.database || tmpSettings.Database,
-					ConnectionPoolLimit: tmpSettings.ConnectionPoolLimit
+					ConnectionPoolLimit: tmpSettings.ConnectionPoolLimit,
+					RequestTimeoutMs: tmpSettings.RequestTimeoutMs,
+					ConnectionTimeoutMs: tmpSettings.ConnectionTimeoutMs,
+					ConnectRetryOptions: tmpSettings.ConnectRetryOptions,
+					DDLRetryOptions: tmpSettings.DDLRetryOptions,
+					LegacyPagination: tmpSettings.LegacyPagination
 				});
 		}
 
 		// Schema provider handles DDL operations (create, drop, index, etc.)
+		// Give it a back-reference so it can trigger pool recycling via the
+		// retry helper when it detects a degraded pool.
 		this._SchemaProvider = new libMeadowSchemaMSSQL(this.fable, this.options, `${this.Hash}-Schema`);
+		this._SchemaProvider.setConnectionProvider(this);
 	}
 
 	get schemaProvider()
@@ -187,29 +214,45 @@ class MeadowConnectionMSSQL extends libFableServiceProviderBase
 		this.connectAsync();
 	}
 
-	connectAsync(fCallback)
+	/**
+	 * Build a node-mssql connection settings object from this provider's
+	 * configured options.  Centralised so both connectAsync and
+	 * recyclePool produce identical settings.
+	 *
+	 * @returns {Object}
+	 */
+	_buildConnectionSettings()
 	{
-		let tmpCallback = fCallback;
-		if (typeof (tmpCallback) !== 'function')
-		{
-			this.log.error(`Meadow MSSQL connect() called without a callback; this could lead to connection race conditions.`);
-			tmpCallback = () => { };
-		}
 		let tmpMSSQLSettings = this.options.MSSQL || {};
-		let tmpConnectionSettings = (
+
+		// Timeouts are configurable — slow WAN links / firewalled customer
+		// networks may need much longer than the driver defaults.
+		let tmpRequestTimeoutMs = tmpMSSQLSettings.RequestTimeoutMs
+			|| tmpMSSQLSettings.requestTimeoutMs
+			|| DEFAULT_REQUEST_TIMEOUT_MS;
+		let tmpConnectionTimeoutMs = tmpMSSQLSettings.ConnectionTimeoutMs
+			|| tmpMSSQLSettings.connectionTimeoutMs
+			|| DEFAULT_CONNECTION_TIMEOUT_MS;
+
+		return (
 			{
 				server: tmpMSSQLSettings.server,
 				user: tmpMSSQLSettings.user,
 				password: tmpMSSQLSettings.password,
 				database: tmpMSSQLSettings.database,
-				requestTimeout: 80000,
-				connectionTimeout: 80000,
+				requestTimeout: tmpRequestTimeoutMs,
+				connectionTimeout: tmpConnectionTimeoutMs,
 				port: tmpMSSQLSettings.port,
 				pool:
 				{
 					max: tmpMSSQLSettings.ConnectionPoolLimit || 10,
 					min: 0,
-					idleTimeoutMillis: 30000
+					idleTimeoutMillis: 30000,
+					// Cap how long pool.query() waits for an available
+					// connection before giving up with a ResourceRequest
+					// timeout.  Default matches connectionTimeout so the
+					// error classifier sees a recognizable shape.
+					acquireTimeoutMillis: tmpConnectionTimeoutMs
 				},
 				options:
 				{
@@ -217,34 +260,141 @@ class MeadowConnectionMSSQL extends libFableServiceProviderBase
 					trustServerCertificate: true // change to true for local dev / self-signed customer certs
 				},
 			});
+	}
+
+	/**
+	 * Resolve retry options from this provider's config, falling back to
+	 * sensible defaults tuned for DDL over a slow network.
+	 *
+	 * @returns {Object}
+	 */
+	_connectRetryOptions()
+	{
+		let tmpMSSQLSettings = this.options.MSSQL || {};
+		let tmpRetry = tmpMSSQLSettings.ConnectRetryOptions || {};
+		return (
+			{
+				OperationName: `Meadow-MSSQL connect to [${tmpMSSQLSettings.server}:${tmpMSSQLSettings.port || 1433}]`,
+				MaxAttempts: tmpRetry.MaxAttempts || DEFAULT_CONNECT_MAX_ATTEMPTS,
+				InitialDelayMs: tmpRetry.InitialDelayMs || DEFAULT_CONNECT_INITIAL_DELAY,
+				MaxDelayMs: tmpRetry.MaxDelayMs || DEFAULT_CONNECT_MAX_DELAY,
+				BackoffFactor: tmpRetry.BackoffFactor || 2,
+				// Connection-establishment retries never recycle a pool
+				// (there isn't one yet).  OnRecyclePool is intentionally
+				// absent here.
+			});
+	}
+
+	connectAsync(fCallback)
+	{
+		let tmpCallback = fCallback;
+		if (typeof (tmpCallback) !== 'function')
+		{
+			this.log.error(`Meadow MSSQL connect() called without a callback; this could lead to connection race conditions.`);
+			tmpCallback = () => { };
+		}
+
 		if (this._ConnectionPool)
 		{
-			tmpCleansedLogSettings = JSON.parse(JSON.stringify(tmpConnectionSettings));
+			let tmpCleansedLogSettings = JSON.parse(JSON.stringify(this._buildConnectionSettings()));
 			// No leaking passwords!
 			tmpCleansedLogSettings.password = '*****************';
 			this.log.error(`Meadow-Connection-MSSQL trying to connect to MSSQL but is already connected - skipping the generation of extra connections.`, tmpCleansedLogSettings);
 			return tmpCallback(null, this._ConnectionPool);
 		}
-		else
+
+		let tmpConnectionSettings = this._buildConnectionSettings();
+		this.log.info(`Meadow-Connection-MSSQL connecting to [${tmpConnectionSettings.server} : ${tmpConnectionSettings.port}] as ${tmpConnectionSettings.user} for database ${tmpConnectionSettings.database} at a connection limit of ${tmpConnectionSettings.pool.max} (connectionTimeout: ${(tmpConnectionSettings.connectionTimeout / 1000).toFixed(0)}s, requestTimeout: ${(tmpConnectionSettings.requestTimeout / 1000).toFixed(0)}s)`);
+
+		// Retry with exponential backoff on transient connect failures
+		// (network errors, timeouts).  Hard failures like authentication
+		// errors get classified as ServerError and propagate immediately.
+		libRetry.runWithRetry(this.log, this._connectRetryOptions(),
+			(fAttemptDone) =>
+			{
+				libMSSQL.connect(tmpConnectionSettings)
+					.then((pConnectionPool) => fAttemptDone(null, pConnectionPool))
+					.catch((pError) => fAttemptDone(pError));
+			},
+			(pError, pConnectionPool) =>
+			{
+				if (pError)
+				{
+					this.log.error(`Meadow-Connection-MSSQL final connection failure to [${tmpConnectionSettings.server} : ${tmpConnectionSettings.port}] as ${tmpConnectionSettings.user} for database ${tmpConnectionSettings.database}: ${libRetry.extractErrorMessage(pError)}`);
+					return tmpCallback(pError);
+				}
+
+				this.log.info(`Meadow-Connection-MSSQL successfully connected to MSSQL at [${tmpConnectionSettings.server} : ${tmpConnectionSettings.port}] as ${tmpConnectionSettings.user} for database ${tmpConnectionSettings.database} at a connection limit of ${tmpConnectionSettings.pool.max}.`);
+				this._ConnectionPool = pConnectionPool;
+				this.connected = true;
+				this._SchemaProvider.setConnectionPool(this._ConnectionPool);
+				return tmpCallback(null, this._ConnectionPool);
+			});
+	}
+
+	/**
+	 * Destroy the current pool and create a fresh one.  Used by the retry
+	 * helper when a failure mode (RequestTimeout, PoolDegraded) suggests
+	 * the pooled connection is in a bad state that new queries on the
+	 * same pool won't recover from.
+	 *
+	 * Idempotent: safe to call even if there is no pool.  Always resolves
+	 * (even on error) so callers can proceed with the retry regardless.
+	 *
+	 * @param {Function} fCallback - (err) => ... (err is informational only)
+	 */
+	recyclePool(fCallback)
+	{
+		let tmpCallback = (typeof (fCallback) === 'function') ? fCallback : () => {};
+
+		this.log.info(`Meadow-Connection-MSSQL: recycling connection pool...`);
+
+		// Close the existing pool.  mssql's pool.close() is a promise; handle
+		// both shapes (some versions are callback-based).
+		let fCloseAndReconnect = () =>
 		{
-			this.log.info(`Meadow-Connection-MSSQL connecting to [${tmpConnectionSettings.server} : ${tmpConnectionSettings.port}] as ${tmpConnectionSettings.user} for database ${tmpConnectionSettings.database} at a connection limit of ${tmpConnectionSettings.pool.max}`);
-			libMSSQL.connect(tmpConnectionSettings)
-				.then(
-					(pConnectionPool) =>
-					{
-						this.log.info(`Meadow-Connection-MSSQL successfully connected to MSSQL at [${tmpConnectionSettings.server} : ${tmpConnectionSettings.port}] as ${tmpConnectionSettings.user} for database ${tmpConnectionSettings.database} at a connection limit of ${tmpConnectionSettings.pool.max}.`);
-						this._ConnectionPool = pConnectionPool;
-						this.connected = true;
-						this._SchemaProvider.setConnectionPool(this._ConnectionPool);
-						return tmpCallback(null, this._ConnectionPool)
-					})
-				.catch(
-					(pError) =>
+			this._ConnectionPool = false;
+			this.connected = false;
+			this._SchemaProvider.setConnectionPool(false);
+			this.connectAsync((pConnectError) =>
+			{
+				if (pConnectError)
+				{
+					this.log.warn(`Meadow-Connection-MSSQL: pool recycle reconnect failed — ${libRetry.extractErrorMessage(pConnectError)}`);
+					return tmpCallback(pConnectError);
+				}
+				this.log.info(`Meadow-Connection-MSSQL: pool recycle complete.`);
+				return tmpCallback();
+			});
+		};
+
+		if (this._ConnectionPool && typeof (this._ConnectionPool.close) === 'function')
+		{
+			let tmpCloseResult;
+			try
+			{
+				tmpCloseResult = this._ConnectionPool.close();
+			}
+			catch (pCloseError)
+			{
+				this.log.warn(`Meadow-Connection-MSSQL: pool.close() threw — ${pCloseError.message || pCloseError} — continuing with reconnect anyway`);
+				return fCloseAndReconnect();
+			}
+
+			if (tmpCloseResult && typeof (tmpCloseResult.then) === 'function')
+			{
+				tmpCloseResult
+					.then(() => fCloseAndReconnect())
+					.catch((pCloseError) =>
 					{
-						this.log.error(`Meadow-Connection-MSSQL error connecting to MSSQL at [${tmpConnectionSettings.server} : ${tmpConnectionSettings.port}] as ${tmpConnectionSettings.user} for database ${tmpConnectionSettings.database} at a connection limit of ${tmpConnectionSettings.pool.max}.`, pError);
-						return tmpCallback(pError);
+						this.log.warn(`Meadow-Connection-MSSQL: pool.close() rejected — ${pCloseError.message || pCloseError} — continuing with reconnect anyway`);
+						fCloseAndReconnect();
 					});
+				return;
+			}
 		}
+
+		fCloseAndReconnect();
 	}
 
 	get preparedStatement()

package/source/Meadow-MSSQL-Retry.js

@@ -0,0 +1,347 @@
+/**
+ * Meadow MSSQL Retry + Error Classification
+ *
+ * Shared helper used by both the connection provider (Meadow-Connection-MSSQL)
+ * and the schema provider (Meadow-Schema-MSSQL) to:
+ *
+ *   1. Classify MSSQL driver errors into a small set of well-known failure
+ *      modes, so logs make it obvious WHY an operation failed (network,
+ *      lock contention, pool degradation, etc.).
+ *   2. Retry transient operations with exponential backoff, logging each
+ *      attempt in a way that makes the failure mode sequence readable.
+ *   3. Optionally recycle the connection pool on failures that suggest the
+ *      pooled connection is in a bad state (timeouts, network drops).
+ *
+ * Each failure mode is described with a human-readable string so a
+ * developer reading a sync log can tell at a glance whether the problem
+ * is "slow network", "server-side DDL lock", or "pool went bad after
+ * timeout" — these are very different things and each has a different
+ * resolution path.
+ *
+ * @license MIT
+ * @author Steven Velozo <steven@velozo.com>
+ */
+
+'use strict';
+
+/**
+ * Default retry options.  Callers may override any of these per-operation.
+ *
+ *   MaxAttempts        — total attempts including the first.  5 means
+ *                        "first try + up to 4 retries".
+ *   InitialDelayMs     — wait before the second attempt.  Subsequent
+ *                        retries use exponential backoff from here.
+ *   MaxDelayMs         — cap the backoff so we don't wait absurdly long.
+ *   BackoffFactor      — multiplier between retries (2 = double each time).
+ *   PoolRecycleModes   — failure modes that trigger pool recycling before
+ *                        the next attempt.  Kept explicit so callers can
+ *                        narrow or widen the set without reading code.
+ */
+const DEFAULT_RETRY_OPTIONS =
+{
+	MaxAttempts: 5,
+	InitialDelayMs: 2000,
+	MaxDelayMs: 30000,
+	BackoffFactor: 2,
+	PoolRecycleModes: ['NetworkError', 'RequestTimeout', 'PoolDegraded']
+};
+
+/**
+ * Error-mode strings.  Exported so callers can reference them in
+ * configuration (e.g. extending PoolRecycleModes) without stringly-typing.
+ */
+const ERROR_MODES =
+{
+	AlreadyExists:  'AlreadyExists',
+	NetworkError:   'NetworkError',
+	RequestTimeout: 'RequestTimeout',
+	PoolDegraded:   'PoolDegraded',
+	ServerError:    'ServerError',
+	Unknown:        'Unknown'
+};
+
+/**
+ * Extract the best available diagnostic string from a node-mssql error.
+ * The driver stashes the real server message in different places depending
+ * on the failure path — probe all of them and concatenate what we find.
+ *
+ * @param {Error} pError
+ * @return {string}
+ */
+function extractErrorMessage(pError)
+{
+	if (!pError) return '';
+
+	let tmpTop = pError.message || '';
+
+	let tmpInner = '';
+	if (pError.originalError)
+	{
+		if (pError.originalError.info && pError.originalError.info.message)
+		{
+			tmpInner = pError.originalError.info.message;
+		}
+		else if (pError.originalError.message)
+		{
+			tmpInner = pError.originalError.message;
+		}
+	}
+
+	if (tmpInner && tmpInner !== tmpTop)
+	{
+		return `${tmpTop} — ${tmpInner}`;
+	}
+	return tmpTop;
+}
+
+/**
+ * Classify a node-mssql error into one of the well-known failure modes.
+ *
+ * @param {Error}  pError       — the error from the mssql driver
+ * @param {number} pElapsedMs   — how long the failed attempt took
+ * @param {Object} [pPriorInfo] — { lastFailureMode, lastFailureElapsed }
+ *                                from the most recent prior attempt; used
+ *                                to detect the fast-fail-after-timeout
+ *                                pattern that signals pool degradation.
+ *
+ * @return {{
+ *   mode: string,
+ *   description: string,
+ *   isRetryable: boolean,
+ *   recommendPoolRecycle: boolean
+ * }}
+ */
+function classifyError(pError, pElapsedMs, pPriorInfo)
+{
+	let tmpCode = (pError && pError.code) || '';
+	let tmpMsg = extractErrorMessage(pError);
+	let tmpMsgLower = tmpMsg.toLowerCase();
+
+	// Benign case — target already exists.  Callers should treat as success.
+	if (tmpMsgLower.indexOf('there is already an object named') >= 0)
+	{
+		return {
+			mode: ERROR_MODES.AlreadyExists,
+			description: 'target already exists (benign on re-deploy)',
+			isRetryable: false,
+			recommendPoolRecycle: false
+		};
+	}
+
+	// Server-level error (syntax, permissions, constraint violations).
+	// These are deterministic — retrying won't help.  Checked early so we
+	// don't misclassify a server message as a timeout just because it
+	// contains the word "timeout" somewhere coincidentally.
+	if (tmpCode === 'EREQUEST' && tmpMsgLower.indexOf('timeout') < 0)
+	{
+		return {
+			mode: ERROR_MODES.ServerError,
+			description: `server-side SQL error: ${tmpMsg}`,
+			isRetryable: false,
+			recommendPoolRecycle: false
+		};
+	}
+
+	// Pool-degraded heuristic: a fast failure (<500ms) right after a
+	// prior slow failure (>5s) almost always means the pooled connection
+	// is in a bad state — the server was slow, the socket went stale,
+	// and the next query on the same pool died fast.  Check this BEFORE
+	// the generic NetworkError/RequestTimeout branches so the specific
+	// mode wins in the log.
+	if (pElapsedMs < 500 &&
+		pPriorInfo &&
+		pPriorInfo.lastFailureMode === ERROR_MODES.RequestTimeout &&
+		pPriorInfo.lastFailureElapsed > 5000)
+	{
+		return {
+			mode: ERROR_MODES.PoolDegraded,
+			description: `query failed in ${pElapsedMs}ms after prior ${(pPriorInfo.lastFailureElapsed / 1000).toFixed(1)}s timeout — connection pool appears degraded (stale/bad connection)`,
+			isRetryable: true,
+			recommendPoolRecycle: true
+		};
+	}
+
+	// OS-level socket timeout (ETIMEDOUT) — the kernel gave up trying to
+	// complete the TCP operation.  Treat as a network failure.
+	if (tmpCode === 'ETIMEDOUT')
+	{
+		return {
+			mode: ERROR_MODES.NetworkError,
+			description: `network-level timeout (${tmpCode}) — socket didn't complete in OS-level timeout: ${tmpMsg}`,
+			isRetryable: true,
+			recommendPoolRecycle: true
+		};
+	}
+
+	// Network-level failures — server unreachable, DNS, socket dropped.
+	// These all indicate the problem is below the SQL layer.
+	if (tmpCode === 'ECONNREFUSED' ||
+		tmpCode === 'ECONNRESET' ||
+		tmpCode === 'ENETUNREACH' ||
+		tmpCode === 'EHOSTUNREACH' ||
+		tmpCode === 'ENOTFOUND' ||
+		tmpCode === 'EAI_AGAIN' ||
+		tmpCode === 'ESOCKET')
+	{
+		return {
+			mode: ERROR_MODES.NetworkError,
+			description: `network-level failure (${tmpCode}) — server unreachable or connection dropped: ${tmpMsg}`,
+			isRetryable: true,
+			recommendPoolRecycle: true
+		};
+	}
+
+	// Request timeout — the query reached the server but didn't respond in
+	// time.  On DDL operations this almost always means another transaction
+	// is holding an exclusive schema lock.  On data queries it could mean a
+	// long-running query is blocking, or the server is overloaded.  Note
+	// the code is `ETIMEOUT` (no D), which is node-mssql / tedious's own
+	// request-timeout code, distinct from the kernel-level ETIMEDOUT above.
+	if (tmpCode === 'ETIMEOUT' ||
+		tmpMsgLower.indexOf('operation timed out') >= 0 ||
+		tmpMsgLower.indexOf('request timeout') >= 0 ||
+		tmpMsgLower.indexOf('timed out for an unknown reason') >= 0)
+	{
+		return {
+			mode: ERROR_MODES.RequestTimeout,
+			description: `request timed out after ${(pElapsedMs / 1000).toFixed(1)}s — likely DDL lock contention on the server, a long-running blocking query, or a very slow server response`,
+			isRetryable: true,
+			recommendPoolRecycle: true
+		};
+	}
+
+	// Everything else — retry cautiously (it might be transient) but don't
+	// recycle the pool since we don't know what kind of failure it is.
+	return {
+		mode: ERROR_MODES.Unknown,
+		description: `unclassified error (${tmpCode || 'no code'}): ${tmpMsg}`,
+		isRetryable: true,
+		recommendPoolRecycle: false
+	};
+}
+
+/**
+ * Run an operation with retry, exponential backoff, error classification,
+ * and optional pool recycling.  Callback-style to match the rest of the
+ * Meadow codebase.
+ *
+ * The operation function is called with a single `fDone(err, result)`
+ * callback.  `err` may be null on success.  Anything truthy is classified
+ * and either retried (if the mode is retryable) or propagated.
+ *
+ * Log output is structured so a reader can follow the decision tree:
+ *
+ *   [info]  Meadow-MSSQL CREATE TABLE Sample: attempt 1/5 starting...
+ *   [warn]  Meadow-MSSQL CREATE TABLE Sample: attempt 1/5 failed after 30012ms — request timed out ... (likely DDL lock contention ...)
+ *   [info]  Meadow-MSSQL CREATE TABLE Sample: recycling connection pool before retry (mode: RequestTimeout)
+ *   [info]  Meadow-MSSQL CREATE TABLE Sample: retrying in 2s (attempt 2/5)
+ *   [info]  Meadow-MSSQL CREATE TABLE Sample: attempt 2/5 starting...
+ *   [info]  Meadow-MSSQL CREATE TABLE Sample: succeeded on attempt 2 (elapsed 3542ms)
+ *
+ * @param {Object}   pLog       — a fable logger (has .info, .warn, .error)
+ * @param {Object}   pOptions   — { OperationName, MaxAttempts, InitialDelayMs, MaxDelayMs, BackoffFactor, PoolRecycleModes, OnRecyclePool, SuccessModes }
+ * @param {Function} fOperation — (fDone) => ... where fDone(err, result)
+ * @param {Function} fCallback  — (err, result) => ...
+ */
+function runWithRetry(pLog, pOptions, fOperation, fCallback)
+{
+	let tmpOptions = Object.assign({}, DEFAULT_RETRY_OPTIONS, pOptions || {});
+	let tmpOpName = tmpOptions.OperationName || 'Meadow-MSSQL operation';
+	let tmpPriorInfo = null;
+	let tmpAttempt = 0;
+
+	// SuccessModes lets callers treat certain failure modes as success
+	// (e.g. AlreadyExists for CREATE TABLE — not really an error).
+	let tmpSuccessModes = tmpOptions.SuccessModes || [ERROR_MODES.AlreadyExists];
+
+	let fTryOnce = () =>
+	{
+		tmpAttempt++;
+		let tmpStartMs = Date.now();
+
+		pLog.info(`${tmpOpName}: attempt ${tmpAttempt}/${tmpOptions.MaxAttempts} starting...`);
+
+		fOperation((pError, pResult) =>
+		{
+			let tmpElapsedMs = Date.now() - tmpStartMs;
+
+			if (!pError)
+			{
+				if (tmpAttempt > 1)
+				{
+					pLog.info(`${tmpOpName}: succeeded on attempt ${tmpAttempt} (elapsed ${tmpElapsedMs}ms)`);
+				}
+				return fCallback(null, pResult);
+			}
+
+			let tmpInfo = classifyError(pError, tmpElapsedMs, tmpPriorInfo);
+
+			// Treat success-mode failures as success.
+			if (tmpSuccessModes.indexOf(tmpInfo.mode) >= 0)
+			{
+				pLog.info(`${tmpOpName}: ${tmpInfo.description} — treating as success`);
+				return fCallback(null, pResult);
+			}
+
+			pLog.warn(`${tmpOpName}: attempt ${tmpAttempt}/${tmpOptions.MaxAttempts} failed after ${tmpElapsedMs}ms [${tmpInfo.mode}] — ${tmpInfo.description}`);
+
+			tmpPriorInfo = { lastFailureMode: tmpInfo.mode, lastFailureElapsed: tmpElapsedMs };
+
+			// Terminal: non-retryable or exhausted attempts.
+			if (!tmpInfo.isRetryable)
+			{
+				pLog.error(`${tmpOpName}: giving up — ${tmpInfo.mode} is not retryable`);
+				return fCallback(pError);
+			}
+			if (tmpAttempt >= tmpOptions.MaxAttempts)
+			{
+				pLog.error(`${tmpOpName}: giving up — exhausted ${tmpOptions.MaxAttempts} attempts; last failure mode was ${tmpInfo.mode}`);
+				return fCallback(pError);
+			}
+
+			// Compute next delay with exponential backoff, capped at MaxDelayMs.
+			let tmpDelayMs = Math.min(
+				tmpOptions.InitialDelayMs * Math.pow(tmpOptions.BackoffFactor, tmpAttempt - 1),
+				tmpOptions.MaxDelayMs);
+
+			let fScheduleRetry = () =>
+			{
+				pLog.info(`${tmpOpName}: retrying in ${(tmpDelayMs / 1000).toFixed(1)}s (attempt ${tmpAttempt + 1}/${tmpOptions.MaxAttempts})`);
+				setTimeout(fTryOnce, tmpDelayMs);
+			};
+
+			// Recycle the pool if the failure mode suggests it AND the caller
+			// provided a recycling hook.  Pool recycling is async too — wait
+			// for it to finish before scheduling the retry.
+			let tmpShouldRecycle = tmpOptions.PoolRecycleModes.indexOf(tmpInfo.mode) >= 0
+				&& tmpInfo.recommendPoolRecycle
+				&& typeof (tmpOptions.OnRecyclePool) === 'function';
+
+			if (tmpShouldRecycle)
+			{
+				pLog.info(`${tmpOpName}: recycling connection pool before retry (mode: ${tmpInfo.mode})`);
+				tmpOptions.OnRecyclePool((pRecycleError) =>
+				{
+					if (pRecycleError)
+					{
+						pLog.warn(`${tmpOpName}: pool recycle failed — ${pRecycleError.message || pRecycleError}; retrying anyway`);
+					}
+					fScheduleRetry();
+				});
+				return;
+			}
+
+			fScheduleRetry();
+		});
+	};
+
+	fTryOnce();
+}
+
+module.exports = {
+	classifyError: classifyError,
+	runWithRetry: runWithRetry,
+	extractErrorMessage: extractErrorMessage,
+	DEFAULT_RETRY_OPTIONS: DEFAULT_RETRY_OPTIONS,
+	ERROR_MODES: ERROR_MODES
+};

package/source/Meadow-Schema-MSSQL.js

@@ -9,6 +9,16 @@
 */
 const libFableServiceProviderBase = require('fable-serviceproviderbase');
 
+const libRetry = require('./Meadow-MSSQL-Retry.js');
+
+// Default retry behavior for DDL operations.  CREATE TABLE and CREATE
+// INDEX against a heavily-used MSSQL instance can hit schema locks held
+// by unrelated transactions; retry with exponential backoff so a busy
+// server window doesn't kill a whole deploy.
+const DEFAULT_DDL_MAX_ATTEMPTS   = 5;
+const DEFAULT_DDL_INITIAL_DELAY  = 3000;
+const DEFAULT_DDL_MAX_DELAY      = 30000;
+
 class MeadowSchemaMSSQL extends libFableServiceProviderBase
 {
 	constructor(pFable, pOptions, pServiceHash)
@@ -19,6 +29,11 @@ class MeadowSchemaMSSQL extends libFableServiceProviderBase
 
 		// Reference to the connection pool, set by the connection provider
 		this._ConnectionPool = false;
+
+		// Back-reference to the MeadowConnectionMSSQL that owns this
+		// schema provider.  Used to request a pool recycle when a DDL
+		// failure mode suggests the pooled connection is in a bad state.
+		this._ConnectionProvider = null;
 	}
 
 	/**
@@ -32,6 +47,67 @@ class MeadowSchemaMSSQL extends libFableServiceProviderBase
 		return this;
 	}
 
+	/**
+	 * Set the back-reference to the connection provider.  The retry
+	 * helper uses this to trigger pool recycling on pool-degraded errors.
+	 *
+	 * @param {object} pConnectionProvider - MeadowConnectionMSSQL instance
+	 * @returns {MeadowSchemaMSSQL} this (for chaining)
+	 */
+	setConnectionProvider(pConnectionProvider)
+	{
+		this._ConnectionProvider = pConnectionProvider;
+		return this;
+	}
+
+	/**
+	 * Build the retry options block used by DDL operations.  Honors
+	 * per-provider overrides via options.MSSQL.DDLRetryOptions.
+	 *
+	 * @param {string} pOperationName - name to use in log output
+	 * @returns {Object}
+	 */
+	_ddlRetryOptions(pOperationName)
+	{
+		let tmpMSSQLSettings = this.options.MSSQL || this.fable.settings.MSSQL || {};
+		let tmpRetry = tmpMSSQLSettings.DDLRetryOptions || {};
+
+		let tmpOptions = (
+			{
+				OperationName: pOperationName,
+				MaxAttempts: tmpRetry.MaxAttempts || DEFAULT_DDL_MAX_ATTEMPTS,
+				InitialDelayMs: tmpRetry.InitialDelayMs || DEFAULT_DDL_INITIAL_DELAY,
+				MaxDelayMs: tmpRetry.MaxDelayMs || DEFAULT_DDL_MAX_DELAY,
+				BackoffFactor: tmpRetry.BackoffFactor || 2,
+				// "AlreadyExists" is always treated as success for DDL — a
+				// re-deploy will naturally hit tables that already exist.
+				SuccessModes: [libRetry.ERROR_MODES.AlreadyExists]
+			});
+
+		// Connect the pool recycle hook if we know how to reach the
+		// connection provider (we always do when set up via
+		// Meadow-Connection-MSSQL, but test harnesses sometimes wire the
+		// schema provider standalone).
+		if (this._ConnectionProvider && typeof (this._ConnectionProvider.recyclePool) === 'function')
+		{
+			tmpOptions.OnRecyclePool = (fRecycleDone) =>
+			{
+				this._ConnectionProvider.recyclePool((pErr) =>
+				{
+					// Refresh our pool reference from the connection provider
+					// after the recycle so the next attempt uses the fresh pool.
+					if (this._ConnectionProvider.pool)
+					{
+						this._ConnectionPool = this._ConnectionProvider.pool;
+					}
+					return fRecycleDone(pErr);
+				});
+			};
+		}
+
+		return tmpOptions;
+	}
+
 	generateDropTableStatement(pTableName)
 	{
 		let tmpDropTableStatement = `IF OBJECT_ID('dbo.[${pTableName}]', 'U') IS NOT NULL\n`;
@@ -42,7 +118,7 @@ class MeadowSchemaMSSQL extends libFableServiceProviderBase
 
 	generateCreateTableStatement(pMeadowTableSchema)
 	{
-		this.log.info(`--> Building the table create string for ${pMeadowTableSchema} ...`);
+		this.log.info(`--> Building the table create string for ${pMeadowTableSchema && pMeadowTableSchema.TableName ? pMeadowTableSchema.TableName : '(unknown)'} ...`);
 
 		let tmpPrimaryKey = false;
 		let tmpCreateTableStatement = `--   [ ${pMeadowTableSchema.TableName} ]`;
@@ -136,31 +212,40 @@ class MeadowSchemaMSSQL extends libFableServiceProviderBase
 
 	createTable(pMeadowTableSchema, fCallback)
 	{
+		let tmpTableName = (pMeadowTableSchema && pMeadowTableSchema.TableName) || '(unknown)';
 		let tmpCreateTableStatement = this.generateCreateTableStatement(pMeadowTableSchema);
-		this._ConnectionPool.query(tmpCreateTableStatement)
-			.then((pResult) =>
+
+		// Wrap the DDL in the retry helper.  The helper classifies errors
+		// and handles:
+		//   - AlreadyExists  → treat as success (benign re-deploy)
+		//   - NetworkError   → exponential backoff, pool recycle
+		//   - RequestTimeout → exponential backoff, pool recycle (covers
+		//                      server-side schema-lock contention)
+		//   - PoolDegraded   → exponential backoff, pool recycle
+		//   - ServerError    → fail fast (syntax/permission errors)
+		//   - Unknown        → exponential backoff, no pool recycle
+		//
+		// Every attempt, failure, and recycle decision is logged so the
+		// operator can see at a glance which failure mode is occurring.
+		libRetry.runWithRetry(this.log,
+			this._ddlRetryOptions(`Meadow-MSSQL CREATE TABLE ${tmpTableName}`),
+			(fAttemptDone) =>
 			{
-				this.log.info(`Meadow-MSSQL CREATE TABLE ${pMeadowTableSchema.TableName} Success`);
-				this.log.warn(`Meadow-MSSQL Create Table Statement: ${tmpCreateTableStatement}`)
-				return fCallback();
-			})
-			.catch((pError) =>
+				this._ConnectionPool.query(tmpCreateTableStatement)
+					.then((pResult) => fAttemptDone(null, pResult))
+					.catch((pError) => fAttemptDone(pError));
+			},
+			(pError) =>
 			{
-				if (pError.hasOwnProperty('originalError')
-					// TODO: This check may be extraneous; not familiar enough with the mssql node driver yet
-					&& (pError.originalError.hasOwnProperty('info'))
-					// TODO: Validate that there isn't a better way to find this (pError.code isn't explicit enough)
-					&& (pError.originalError.info.message.indexOf("There is already an object named") == 0)
-					&& (pError.originalError.info.message.indexOf('in the database.') > 0))
-				{
-					// The table already existed; log a warning but keep on keeping on.
-					return fCallback();
-				}
-				else
+				if (pError)
 				{
-					this.log.error(`Meadow-MSSQL CREATE TABLE ${pMeadowTableSchema.TableName} failed!`, pError);
+					// runWithRetry already logged the final failure with its
+					// classified mode.  Propagate the error without adding
+					// another noisy error line.
 					return fCallback(pError);
 				}
+				this.log.info(`Meadow-MSSQL CREATE TABLE ${tmpTableName} success`);
+				return fCallback();
 			});
 	}
 
@@ -363,35 +448,46 @@ class MeadowSchemaMSSQL extends libFableServiceProviderBase
 			return fCallback(new Error('Not connected to MSSQL'));
 		}
 
-		// First check if the index already exists
-		this._ConnectionPool.query(pIndexStatement.CheckStatement)
-			.then((pCheckResult) =>
+		// Wrap the (check, then create) sequence in the retry helper — on
+		// a flaky connection either query can time out, and the classifier
+		// will surface whether it's network, lock contention, or a stale
+		// pool.  The retry helper will recycle the pool between attempts
+		// when the failure mode recommends it.
+		libRetry.runWithRetry(this.log,
+			this._ddlRetryOptions(`Meadow-MSSQL CREATE INDEX ${pIndexStatement.Name}`),
+			(fAttemptDone) =>
 			{
-				let tmpExists = pCheckResult && pCheckResult.recordset && pCheckResult.recordset[0] && pCheckResult.recordset[0].IndexExists > 0;
-
-				if (tmpExists)
-				{
-					this.log.info(`Meadow-MSSQL INDEX ${pIndexStatement.Name} already exists, skipping.`);
-					return fCallback();
-				}
-
-				// Index does not exist; create it
-				this._ConnectionPool.query(pIndexStatement.Statement)
-					.then(() =>
+				this._ConnectionPool.query(pIndexStatement.CheckStatement)
+					.then((pCheckResult) =>
 					{
-						this.log.info(`Meadow-MSSQL CREATE INDEX ${pIndexStatement.Name} executed successfully.`);
-						return fCallback();
+						let tmpExists = pCheckResult && pCheckResult.recordset && pCheckResult.recordset[0] && pCheckResult.recordset[0].IndexExists > 0;
+						if (tmpExists)
+						{
+							// Signal success with a sentinel result so the
+							// outer callback can log the "already exists" case.
+							return fAttemptDone(null, { AlreadyExisted: true });
+						}
+						this._ConnectionPool.query(pIndexStatement.Statement)
+							.then(() => fAttemptDone(null, { AlreadyExisted: false }))
+							.catch((pCreateError) => fAttemptDone(pCreateError));
 					})
-					.catch((pCreateError) =>
-					{
-						this.log.error(`Meadow-MSSQL CREATE INDEX ${pIndexStatement.Name} failed!`, pCreateError);
-						return fCallback(pCreateError);
-					});
-			})
-			.catch((pCheckError) =>
+					.catch((pCheckError) => fAttemptDone(pCheckError));
+			},
+			(pError, pResult) =>
 			{
-				this.log.error(`Meadow-MSSQL CHECK INDEX ${pIndexStatement.Name} failed!`, pCheckError);
-				return fCallback(pCheckError);
+				if (pError)
+				{
+					return fCallback(pError);
+				}
+				if (pResult && pResult.AlreadyExisted)
+				{
+					this.log.info(`Meadow-MSSQL INDEX ${pIndexStatement.Name} already exists, skipping.`);
+				}
+				else
+				{
+					this.log.info(`Meadow-MSSQL CREATE INDEX ${pIndexStatement.Name} executed successfully.`);
+				}
+				return fCallback();
 			});
 	}
 

package/test/MSSQL-Retry_tests.js

@@ -0,0 +1,351 @@
+/**
+ * Unit tests for the Meadow-MSSQL-Retry helper (classifyError + runWithRetry).
+ *
+ * These are pure-JS tests — no real MSSQL connection needed.  The goal is
+ * to lock in the behavior of the error classifier and the retry loop so
+ * regressions don't silently degrade production observability.
+ *
+ * @license MIT
+ */
+
+'use strict';
+
+const Chai = require('chai');
+const Expect = Chai.expect;
+
+const libRetry = require('../source/Meadow-MSSQL-Retry.js');
+
+// A fake logger that captures every call so we can assert on log output.
+function makeTestLogger()
+{
+	let tmpRecords = [];
+	let tmpLog = (pLevel) => (pMsg) => tmpRecords.push({ level: pLevel, message: String(pMsg) });
+	return {
+		info: tmpLog('info'),
+		warn: tmpLog('warn'),
+		error: tmpLog('error'),
+		records: tmpRecords
+	};
+}
+
+// Helpers that synthesize the shapes real mssql errors take.
+function networkError(pCode) {
+	let e = new Error(`Network failure (${pCode})`);
+	e.code = pCode;
+	return e;
+}
+function timeoutError() {
+	let e = new Error('Request timeout');
+	e.code = 'ETIMEOUT';
+	return e;
+}
+function timeoutUnknownReason() {
+	return new Error('operation timed out for an unknown reason');
+}
+function alreadyExistsError() {
+	let e = new Error('server error');
+	e.code = 'EREQUEST';
+	e.originalError = { info: { message: "There is already an object named 'Sample' in the database." } };
+	return e;
+}
+function serverError(pMsg) {
+	let e = new Error('server error');
+	e.code = 'EREQUEST';
+	e.originalError = { info: { message: pMsg } };
+	return e;
+}
+
+suite('Meadow-MSSQL-Retry',
+	() =>
+	{
+		suite('classifyError',
+			() =>
+			{
+				test('classifies network errors as NetworkError (retryable, recycle)',
+					() =>
+					{
+						let tmpClasses = ['ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'ESOCKET', 'ENOTFOUND'];
+						for (let i = 0; i < tmpClasses.length; i++)
+						{
+							let result = libRetry.classifyError(networkError(tmpClasses[i]), 123);
+							Expect(result.mode).to.equal(libRetry.ERROR_MODES.NetworkError);
+							Expect(result.isRetryable).to.equal(true);
+							Expect(result.recommendPoolRecycle).to.equal(true);
+							Expect(result.description).to.include(tmpClasses[i]);
+						}
+					});
+
+				test('classifies request timeout as RequestTimeout with DDL-lock hint',
+					() =>
+					{
+						let r = libRetry.classifyError(timeoutError(), 30012);
+						Expect(r.mode).to.equal(libRetry.ERROR_MODES.RequestTimeout);
+						Expect(r.isRetryable).to.equal(true);
+						Expect(r.recommendPoolRecycle).to.equal(true);
+						Expect(r.description).to.include('DDL lock contention');
+						Expect(r.description).to.include('30.0s');
+					});
+
+				test('classifies "timed out for an unknown reason" (customer-log string) as RequestTimeout',
+					() =>
+					{
+						let r = libRetry.classifyError(timeoutUnknownReason(), 30000);
+						Expect(r.mode).to.equal(libRetry.ERROR_MODES.RequestTimeout);
+					});
+
+				test('classifies "already exists" as AlreadyExists (non-retryable, no recycle)',
+					() =>
+					{
+						let r = libRetry.classifyError(alreadyExistsError(), 45);
+						Expect(r.mode).to.equal(libRetry.ERROR_MODES.AlreadyExists);
+						Expect(r.isRetryable).to.equal(false);
+						Expect(r.recommendPoolRecycle).to.equal(false);
+					});
+
+				test('classifies generic server error (EREQUEST) as ServerError (non-retryable)',
+					() =>
+					{
+						let r = libRetry.classifyError(serverError('Invalid object name dbo.BadTable'), 50);
+						Expect(r.mode).to.equal(libRetry.ERROR_MODES.ServerError);
+						Expect(r.isRetryable).to.equal(false);
+						Expect(r.description).to.include('Invalid object name');
+					});
+
+				test('detects PoolDegraded pattern (fast-fail after prior slow timeout)',
+					() =>
+					{
+						// Simulate an ECONNRESET happening 150ms after a prior
+						// 30s RequestTimeout — classic stale-pool symptom.
+						let priorInfo = {
+							lastFailureMode: libRetry.ERROR_MODES.RequestTimeout,
+							lastFailureElapsed: 30000
+						};
+						let r = libRetry.classifyError(networkError('ECONNRESET'), 150, priorInfo);
+						Expect(r.mode).to.equal(libRetry.ERROR_MODES.PoolDegraded);
+						Expect(r.isRetryable).to.equal(true);
+						Expect(r.recommendPoolRecycle).to.equal(true);
+						Expect(r.description).to.include('degraded');
+					});
+
+				test('does NOT flag PoolDegraded when prior failure was fast too',
+					() =>
+					{
+						// Two fast failures in a row — not pool degradation,
+						// something deterministic.
+						let priorInfo = {
+							lastFailureMode: libRetry.ERROR_MODES.NetworkError,
+							lastFailureElapsed: 100
+						};
+						let r = libRetry.classifyError(networkError('ECONNRESET'), 200, priorInfo);
+						Expect(r.mode).to.equal(libRetry.ERROR_MODES.NetworkError);
+					});
+
+				test('classifies unknown errors as Unknown, retryable, no recycle',
+					() =>
+					{
+						let e = new Error('something weird');
+						let r = libRetry.classifyError(e, 500);
+						Expect(r.mode).to.equal(libRetry.ERROR_MODES.Unknown);
+						Expect(r.isRetryable).to.equal(true);
+						Expect(r.recommendPoolRecycle).to.equal(false);
+					});
+			});
+
+		suite('extractErrorMessage',
+			() =>
+			{
+				test('returns empty string for null/undefined',
+					() =>
+					{
+						Expect(libRetry.extractErrorMessage(null)).to.equal('');
+						Expect(libRetry.extractErrorMessage(undefined)).to.equal('');
+					});
+
+				test('surfaces inner mssql driver message when present',
+					() =>
+					{
+						let e = new Error('Wrapper');
+						e.originalError = { info: { message: 'The real reason' } };
+						Expect(libRetry.extractErrorMessage(e)).to.include('The real reason');
+						Expect(libRetry.extractErrorMessage(e)).to.include('Wrapper');
+					});
+
+				test('returns top message alone when inner is identical or missing',
+					() =>
+					{
+						let e = new Error('Only one message');
+						Expect(libRetry.extractErrorMessage(e)).to.equal('Only one message');
+					});
+			});
+
+		suite('runWithRetry',
+			() =>
+			{
+				test('calls operation once on success, no retries',
+					(fDone) =>
+					{
+						let log = makeTestLogger();
+						let tmpCalls = 0;
+						libRetry.runWithRetry(log, { OperationName: 'test-op', MaxAttempts: 3 },
+							(fAttemptDone) =>
+							{
+								tmpCalls++;
+								fAttemptDone(null, 'yay');
+							},
+							(err, result) =>
+							{
+								try {
+									Expect(err).to.be.null;
+									Expect(result).to.equal('yay');
+									Expect(tmpCalls).to.equal(1);
+									fDone();
+								} catch (e) { fDone(e); }
+							});
+					});
+
+				test('retries on NetworkError with exponential backoff and succeeds',
+					(fDone) =>
+					{
+						let log = makeTestLogger();
+						let tmpCalls = 0;
+						libRetry.runWithRetry(log,
+							{ OperationName: 'flaky-op', MaxAttempts: 3, InitialDelayMs: 10, MaxDelayMs: 50, BackoffFactor: 2 },
+							(fAttemptDone) =>
+							{
+								tmpCalls++;
+								if (tmpCalls < 3) fAttemptDone(networkError('ECONNRESET'));
+								else fAttemptDone(null, 'finally');
+							},
+							(err, result) =>
+							{
+								try {
+									Expect(err).to.be.null;
+									Expect(result).to.equal('finally');
+									Expect(tmpCalls).to.equal(3);
+									let tmpWarnCount = log.records.filter(r => r.level === 'warn').length;
+									Expect(tmpWarnCount).to.equal(2);
+									fDone();
+								} catch (e) { fDone(e); }
+							});
+					});
+
+				test('logs classified failure mode on each attempt',
+					(fDone) =>
+					{
+						let log = makeTestLogger();
+						libRetry.runWithRetry(log,
+							{ OperationName: 'logtest', MaxAttempts: 2, InitialDelayMs: 5 },
+							(fAttemptDone) => fAttemptDone(networkError('ECONNRESET')),
+							(err) =>
+							{
+								try {
+									Expect(err).to.not.be.null;
+									let tmpRelevant = log.records.filter(r => r.message.indexOf('[NetworkError]') >= 0);
+									Expect(tmpRelevant.length).to.be.at.least(1);
+									fDone();
+								} catch (e) { fDone(e); }
+							});
+					});
+
+				test('does not retry ServerError — fails fast',
+					(fDone) =>
+					{
+						let log = makeTestLogger();
+						let tmpCalls = 0;
+						libRetry.runWithRetry(log,
+							{ OperationName: 'bad-sql', MaxAttempts: 5, InitialDelayMs: 10 },
+							(fAttemptDone) =>
+							{
+								tmpCalls++;
+								fAttemptDone(serverError('Syntax error'));
+							},
+							(err) =>
+							{
+								try {
+									Expect(err).to.not.be.null;
+									Expect(tmpCalls).to.equal(1);
+									let tmpGivingUp = log.records.filter(r => r.message.indexOf('not retryable') >= 0);
+									Expect(tmpGivingUp.length).to.equal(1);
+									fDone();
+								} catch (e) { fDone(e); }
+							});
+					});
+
+				test('treats AlreadyExists as success',
+					(fDone) =>
+					{
+						let log = makeTestLogger();
+						let tmpCalls = 0;
+						libRetry.runWithRetry(log,
+							{ OperationName: 'create-table', MaxAttempts: 3, InitialDelayMs: 10 },
+							(fAttemptDone) =>
+							{
+								tmpCalls++;
+								fAttemptDone(alreadyExistsError());
+							},
+							(err) =>
+							{
+								try {
+									Expect(err).to.be.null;
+									Expect(tmpCalls).to.equal(1);
+									let tmpTreatedAsSuccess = log.records.filter(r => r.message.indexOf('treating as success') >= 0);
+									Expect(tmpTreatedAsSuccess.length).to.equal(1);
+									fDone();
+								} catch (e) { fDone(e); }
+							});
+					});
+
+				test('invokes OnRecyclePool for pool-degrading failure modes',
+					(fDone) =>
+					{
+						let log = makeTestLogger();
+						let tmpRecycleCalls = 0;
+						let tmpCalls = 0;
+						libRetry.runWithRetry(log,
+							{
+								OperationName: 'ddl',
+								MaxAttempts: 3,
+								InitialDelayMs: 5,
+								OnRecyclePool: (fDoneRecycle) =>
+								{
+									tmpRecycleCalls++;
+									setImmediate(fDoneRecycle);
+								}
+							},
+							(fAttemptDone) =>
+							{
+								tmpCalls++;
+								if (tmpCalls === 1) fAttemptDone(timeoutUnknownReason());
+								else fAttemptDone(null, 'ok');
+							},
+							(err) =>
+							{
+								try {
+									Expect(err).to.be.null;
+									Expect(tmpCalls).to.equal(2);
+									Expect(tmpRecycleCalls).to.equal(1);
+									fDone();
+								} catch (e) { fDone(e); }
+							});
+					});
+
+				test('gives up after MaxAttempts and propagates last error',
+					(fDone) =>
+					{
+						let log = makeTestLogger();
+						let tmpLastErr = networkError('ECONNRESET');
+						libRetry.runWithRetry(log,
+							{ OperationName: 'doomed', MaxAttempts: 3, InitialDelayMs: 5 },
+							(fAttemptDone) => fAttemptDone(tmpLastErr),
+							(err) =>
+							{
+								try {
+									Expect(err).to.equal(tmpLastErr);
+									let tmpGiveUp = log.records.filter(r => r.message.indexOf('exhausted 3 attempts') >= 0);
+									Expect(tmpGiveUp.length).to.equal(1);
+									fDone();
+								} catch (e) { fDone(e); }
+							});
+					});
+			});
+	});