@mhmdhammoud/meritt-utils 1.5.1 → 1.5.2

package/ReleaseNotes.md

@@ -1,5 +1,60 @@
 # Changes
 
+## Version 1.5.3
+
+### Logger Improvements - Critical Bug Fixes & Production Reliability
+
+#### Critical Fixes
+
+- **Fixed broken JSON.stringify**: Corrected production log serialization from `JSON.stringify(...args)` to `JSON.stringify(args)` - previously only stringified first argument
+- **Fixed silent failures**: Logger now throws clear errors when `LOG_LEVEL` is invalid instead of silently failing with undefined logger
+- **Fixed return type mismatch**: `getLogger()` now properly returns `PinoLogger` type with explicit error handling
+- **Fixed buffer configuration**: Corrected inconsistent buffer settings to match tests (`minLength: 1024, sync: true`)
+- **Fixed race condition**: Event listener now registered before calling `.end()` to prevent missed flush events on shutdown
+
+#### Environment Variable Validation
+
+- **Added Elasticsearch validation**: Validates required env vars (`ELASTICSEARCH_NODE`, `ELASTICSEARCH_USERNAME`, `ELASTICSEARCH_PASSWORD`, `SERVER_NICKNAME`) with clear error messages
+- **Added integer parsing validation**: All numeric env vars now validated to prevent `NaN` values from breaking configuration
+- **Validates positive numbers**: Ensures flush intervals, buffer sizes, retries, and timeouts are positive integers
+
+#### Configurable Settings (New Environment Variables)
+
+- `ES_FLUSH_INTERVAL_MS` - How often to send logs (default: 2000ms = 2 seconds)
+- `ES_FLUSH_BYTES` - Buffer size before forcing flush (default: 102400 = 100KB)
+- `ES_MAX_RETRIES` - Number of retry attempts on failure (default: 3)
+- `ES_REQUEST_TIMEOUT_MS` - Request timeout before retry (default: 30000ms = 30 seconds)
+
+#### Reliability Improvements
+
+- **Error monitoring**: Elasticsearch transport errors now logged to console instead of silent failure
+  - `error` event: Connection errors with clear messaging
+  - `insertError` event: Document indexing failures
+- **Graceful shutdown**: Properly flushes buffered logs before process exit
+  - Handles `SIGTERM` and `SIGINT` signals
+  - 5-second timeout prevents hanging forever
+  - Proper async/await handling to ensure flush completes
+- **Auto-reconnection**: `sniffOnConnectionFault: true` enables automatic reconnection when Elasticsearch nodes fail
+- **Retry logic**: Configurable retry attempts with timeout for failed requests
+
+#### Type Safety
+
+- **Removed `any` types**: All configurations now use proper TypeScript interfaces
+- **Extended ElasticConfig interface**: Added proper types for `maxRetries`, `requestTimeout`, `sniffOnConnectionFault`, and buffer settings
+- **Full compile-time validation**: Type system now validates all configuration options
+
+#### Performance
+
+- **Optimized flush settings**: Default 2-second interval with 100KB buffer balances real-time logs with reliability
+- **Reduced network overhead**: Larger buffer prevents excessive network calls during high-traffic periods
+
+#### Migration Notes
+
+- All changes are backward compatible
+- Default values ensure existing deployments work without changes
+- Optional environment variables allow fine-tuning per environment
+- No breaking changes to the Logger API
+
 ## Version 1.4.1
 
 - Json stringify the log message if its being pushed to Kibana

package/dist/lib/logger.d.ts

@@ -1,5 +1,4 @@
-import { Options as ElasticConfig } from 'pino-elasticsearch';
-import { LOG_LEVEL, LogEvent } from '../types';
+import { LOG_LEVEL, LogEvent, ElasticConfig } from '../types';
 /**
  * Checks if a given log level is valid.
  * @param level - The log level to check.

package/dist/lib/logger.js

@@ -35,43 +35,177 @@ dotenv.config();
  * Pino logger backend - singleton
  */
 let pinoLogger;
+/**
+ * Elasticsearch transport instance - kept for cleanup
+ */
+let esTransport = null;
+/**
+ * Flag to track if shutdown handlers are registered
+ */
+let shutdownHandlersRegistered = false;
+/**
+ * Validates required Elasticsearch environment variables.
+ * @throws Error if required environment variables are missing.
+ */
+function validateElasticsearchEnv() {
+    const required = ['ELASTICSEARCH_NODE', 'ELASTICSEARCH_USERNAME', 'ELASTICSEARCH_PASSWORD', 'SERVER_NICKNAME'];
+    const missing = required.filter(key => !process.env[key]);
+    if (missing.length > 0) {
+        throw new Error(`Missing required Elasticsearch environment variables: ${missing.join(', ')}`);
+    }
+}
+/**
+ * Safely parses an integer from an environment variable.
+ * @param envValue - The environment variable value to parse.
+ * @param defaultValue - The default value to use if parsing fails.
+ * @param varName - The name of the environment variable (for error messages).
+ * @returns The parsed integer or the default value.
+ * @throws Error if the value is not a valid number.
+ */
+function parseIntEnv(envValue, defaultValue, varName) {
+    if (!envValue) {
+        return defaultValue;
+    }
+    const parsed = parseInt(envValue, 10);
+    if (isNaN(parsed)) {
+        throw new Error(`Invalid value for ${varName}: "${envValue}" is not a valid number. Using default: ${defaultValue}`);
+    }
+    if (parsed < 0) {
+        throw new Error(`Invalid value for ${varName}: "${envValue}" must be a positive number. Using default: ${defaultValue}`);
+    }
+    return parsed;
+}
+/**
+ * Registers shutdown handlers to flush logs before process exits.
+ */
+function registerShutdownHandlers() {
+    if (shutdownHandlersRegistered) {
+        return;
+    }
+    let isShuttingDown = false;
+    const flushAndExit = async (signal, exitCode = 0) => {
+        // Prevent multiple shutdown attempts
+        if (isShuttingDown) {
+            return;
+        }
+        isShuttingDown = true;
+        if (esTransport) {
+            try {
+                // Flush any pending logs
+                await new Promise((resolve) => {
+                    const timeout = setTimeout(() => {
+                        console.error(`[Logger] Flush timeout after 5s on ${signal}`);
+                        resolve();
+                    }, 5000); // 5 second timeout
+                    // Register listener BEFORE calling end() to avoid race condition
+                    esTransport === null || esTransport === void 0 ? void 0 : esTransport.once('finish', () => {
+                        clearTimeout(timeout);
+                        resolve();
+                    });
+                    // Now trigger the flush
+                    esTransport === null || esTransport === void 0 ? void 0 : esTransport.end();
+                });
+            }
+            catch (error) {
+                console.error(`[Logger] Error flushing logs on ${signal}:`, error);
+            }
+        }
+        process.exit(exitCode);
+    };
+    process.on('SIGTERM', () => {
+        flushAndExit('SIGTERM', 0).catch((err) => {
+            console.error('[Logger] Flush and exit failed:', err);
+            process.exit(1);
+        });
+    });
+    process.on('SIGINT', () => {
+        flushAndExit('SIGINT', 130).catch((err) => {
+            console.error('[Logger] Flush and exit failed:', err);
+            process.exit(1);
+        });
+    });
+    process.on('beforeExit', () => {
+        if (esTransport && !isShuttingDown) {
+            esTransport.end();
+        }
+    });
+    shutdownHandlersRegistered = true;
+}
 /**
  * Creates a Pino logger instance with specified Elasticsearch configuration.
  * @param elasticConfig - Optional Elasticsearch configuration.
  * @returns The Pino logger instance.
+ * @throws Error if LOG_LEVEL is invalid or required environment variables are missing.
  */
 function getLogger(elasticConfig) {
     if (!pinoLogger) {
-        if (isValidLogLevel(process.env.LOG_LEVEL)) {
-            const transports = [];
-            if (process.env.NODE_ENV !== 'local' && process.env.NODE_ENV !== 'test') {
-                const esConfig = {
-                    index: process.env.SERVER_NICKNAME,
-                    node: process.env.ELASTICSEARCH_NODE,
-                    auth: {
-                        username: process.env.ELASTICSEARCH_USERNAME,
-                        password: process.env.ELASTICSEARCH_PASSWORD,
-                    },
-                    flushInterval: 1000,
-                    'flush-bytes': 1000,
-                };
-                if (elasticConfig) {
-                    Object.assign(esConfig, elasticConfig);
-                }
-                const esTransport = (0, pino_elasticsearch_1.default)(esConfig);
-                transports.push(esTransport);
+        // Validate log level - will throw if invalid
+        const logLevel = process.env.LOG_LEVEL || 'info';
+        isValidLogLevel(logLevel);
+        const transports = [];
+        if (process.env.NODE_ENV !== 'local' && process.env.NODE_ENV !== 'test') {
+            // Validate Elasticsearch environment variables
+            validateElasticsearchEnv();
+            // Parse flush settings from environment with validation
+            const flushIntervalMs = parseIntEnv(process.env.ES_FLUSH_INTERVAL_MS, 2000, // Default: 2 seconds
+            'ES_FLUSH_INTERVAL_MS');
+            const flushBytes = parseIntEnv(process.env.ES_FLUSH_BYTES, 100 * 1024, // Default: 100KB
+            'ES_FLUSH_BYTES');
+            const maxRetries = parseIntEnv(process.env.ES_MAX_RETRIES, 3, // Default: 3 retries
+            'ES_MAX_RETRIES');
+            const requestTimeout = parseIntEnv(process.env.ES_REQUEST_TIMEOUT_MS, 30000, // Default: 30 seconds
+            'ES_REQUEST_TIMEOUT_MS');
+            // Safe to access after validation
+            const esConfig = {
+                index: process.env.SERVER_NICKNAME,
+                node: process.env.ELASTICSEARCH_NODE,
+                auth: {
+                    username: process.env.ELASTICSEARCH_USERNAME,
+                    password: process.env.ELASTICSEARCH_PASSWORD,
+                },
+                // Configurable flush settings
+                flushInterval: flushIntervalMs,
+                'flush-bytes': flushBytes,
+                // Retry configuration for connection resilience
+                maxRetries: maxRetries,
+                requestTimeout: requestTimeout,
+                // Automatically reconnect on connection faults
+                sniffOnConnectionFault: true,
+            };
+            if (elasticConfig) {
+                Object.assign(esConfig, elasticConfig);
             }
-            else {
-                transports.push(pino_1.pino.destination({
-                    minLength: 128,
-                    sync: false,
-                }));
-            }
-            pinoLogger = (0, pino_1.pino)({
-                level: process.env.LOG_LEVEL,
-                timestamp: pino_1.stdTimeFunctions.isoTime.bind(pino_1.stdTimeFunctions),
-            }, ...transports);
+            // Create transport and store reference for cleanup
+            // Cast to PinoElasticOptions since our ElasticConfig includes ClientOptions properties
+            esTransport = (0, pino_elasticsearch_1.default)(esConfig);
+            // Handle Elasticsearch connection errors
+            esTransport.on('error', (err) => {
+                console.error('[Logger] Elasticsearch transport error:', err.message);
+                console.error('[Logger] Logs may not be reaching Kibana. Check Elasticsearch connection.');
+            });
+            // Handle insert errors (document indexing failures)
+            esTransport.on('insertError', (err) => {
+                console.error('[Logger] Elasticsearch insert error:', err.message);
+                console.error('[Logger] Some logs failed to index to Elasticsearch.');
+            });
+            // Log successful connection (for debugging)
+            esTransport.on('insert', () => {
+                // Uncomment for debugging: console.log(`[Logger] Successfully inserted log entries`)
+            });
+            // Register shutdown handlers to flush logs on process exit
+            registerShutdownHandlers();
+            transports.push(esTransport);
+        }
+        else {
+            transports.push(pino_1.pino.destination({
+                minLength: 1024,
+                sync: true,
+            }));
         }
+        pinoLogger = (0, pino_1.pino)({
+            level: logLevel,
+            timestamp: pino_1.stdTimeFunctions.isoTime.bind(pino_1.stdTimeFunctions),
+        }, ...transports);
     }
     return pinoLogger;
 }
@@ -102,8 +236,7 @@ class Logger {
             detail = args;
         }
         else {
-            //@ts-ignore
-            detail = JSON.stringify(...args);
+            detail = JSON.stringify(args);
         }
         this._logger[logLevel]({
             component: this._name,

package/dist/types/logger.d.ts

@@ -44,4 +44,21 @@ export interface ElasticConfig {
      * The interval (in milliseconds) at which logs are flushed to Elasticsearch.
      */
     flushInterval?: number;
+    /**
+     * The number of bytes to buffer before flushing to Elasticsearch.
+     */
+    'flush-bytes'?: number;
+    /**
+     * Maximum number of retries for failed Elasticsearch requests.
+     */
+    maxRetries?: number;
+    /**
+     * Request timeout in milliseconds before considering a request failed.
+     */
+    requestTimeout?: number;
+    /**
+     * Whether to sniff for additional Elasticsearch nodes on connection fault.
+     * Enables automatic reconnection when a node fails.
+     */
+    sniffOnConnectionFault?: boolean;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@mhmdhammoud/meritt-utils",
-	"version": "1.5.1",
+	"version": "1.5.2",
 	"description": "",
 	"main": "./dist/index.js",
 	"private": false,

package/src/lib/logger.ts

@@ -1,7 +1,7 @@
 import {Logger as PinoLogger, pino, stdTimeFunctions} from 'pino'
 import * as dotenv from 'dotenv'
-import pinoElastic, {Options as ElasticConfig} from 'pino-elasticsearch'
-import {LOG_LEVEL, LogEvent} from '../types'
+import pinoElastic, {Options as PinoElasticOptions} from 'pino-elasticsearch'
+import {LOG_LEVEL, LogEvent, ElasticConfig} from '../types'
 
 dotenv.config()
 
@@ -10,50 +10,230 @@ dotenv.config()
  */
 let pinoLogger: PinoLogger
 
+/**
+ * Elasticsearch transport instance - kept for cleanup
+ */
+let esTransport: ReturnType<typeof pinoElastic> | null = null
+
+/**
+ * Flag to track if shutdown handlers are registered
+ */
+let shutdownHandlersRegistered = false
+
+/**
+ * Validates required Elasticsearch environment variables.
+ * @throws Error if required environment variables are missing.
+ */
+function validateElasticsearchEnv(): void {
+	const required = ['ELASTICSEARCH_NODE', 'ELASTICSEARCH_USERNAME', 'ELASTICSEARCH_PASSWORD', 'SERVER_NICKNAME']
+	const missing = required.filter(key => !process.env[key])
+
+	if (missing.length > 0) {
+		throw new Error(
+			`Missing required Elasticsearch environment variables: ${missing.join(', ')}`
+		)
+	}
+}
+
+/**
+ * Safely parses an integer from an environment variable.
+ * @param envValue - The environment variable value to parse.
+ * @param defaultValue - The default value to use if parsing fails.
+ * @param varName - The name of the environment variable (for error messages).
+ * @returns The parsed integer or the default value.
+ * @throws Error if the value is not a valid number.
+ */
+function parseIntEnv(envValue: string | undefined, defaultValue: number, varName: string): number {
+	if (!envValue) {
+		return defaultValue
+	}
+
+	const parsed = parseInt(envValue, 10)
+
+	if (isNaN(parsed)) {
+		throw new Error(
+			`Invalid value for ${varName}: "${envValue}" is not a valid number. Using default: ${defaultValue}`
+		)
+	}
+
+	if (parsed < 0) {
+		throw new Error(
+			`Invalid value for ${varName}: "${envValue}" must be a positive number. Using default: ${defaultValue}`
+		)
+	}
+
+	return parsed
+}
+
+/**
+ * Registers shutdown handlers to flush logs before process exits.
+ */
+function registerShutdownHandlers(): void {
+	if (shutdownHandlersRegistered) {
+		return
+	}
+
+	let isShuttingDown = false
+
+	const flushAndExit = async (signal: string, exitCode = 0) => {
+		// Prevent multiple shutdown attempts
+		if (isShuttingDown) {
+			return
+		}
+		isShuttingDown = true
+
+		if (esTransport) {
+			try {
+				// Flush any pending logs
+				await new Promise<void>((resolve) => {
+					const timeout = setTimeout(() => {
+						console.error(`[Logger] Flush timeout after 5s on ${signal}`)
+						resolve()
+					}, 5000) // 5 second timeout
+
+					// Register listener BEFORE calling end() to avoid race condition
+					esTransport?.once('finish', () => {
+						clearTimeout(timeout)
+						resolve()
+					})
+
+					// Now trigger the flush
+					esTransport?.end()
+				})
+			} catch (error) {
+				console.error(`[Logger] Error flushing logs on ${signal}:`, error)
+			}
+		}
+		process.exit(exitCode)
+	}
+
+	process.on('SIGTERM', () => {
+		flushAndExit('SIGTERM', 0).catch((err) => {
+			console.error('[Logger] Flush and exit failed:', err)
+			process.exit(1)
+		})
+	})
+
+	process.on('SIGINT', () => {
+		flushAndExit('SIGINT', 130).catch((err) => {
+			console.error('[Logger] Flush and exit failed:', err)
+			process.exit(1)
+		})
+	})
+
+	process.on('beforeExit', () => {
+		if (esTransport && !isShuttingDown) {
+			esTransport.end()
+		}
+	})
+
+	shutdownHandlersRegistered = true
+}
+
 /**
  * Creates a Pino logger instance with specified Elasticsearch configuration.
  * @param elasticConfig - Optional Elasticsearch configuration.
  * @returns The Pino logger instance.
+ * @throws Error if LOG_LEVEL is invalid or required environment variables are missing.
  */
 function getLogger(elasticConfig?: ElasticConfig): PinoLogger {
 	if (!pinoLogger) {
-		if (isValidLogLevel(process.env.LOG_LEVEL)) {
-			const transports = []
-			if (process.env.NODE_ENV !== 'local' && process.env.NODE_ENV !== 'test') {
-				const esConfig: ElasticConfig = {
-					index: process.env.SERVER_NICKNAME,
-					node: process.env.ELASTICSEARCH_NODE,
-					auth: {
-						username: process.env.ELASTICSEARCH_USERNAME,
-						password: process.env.ELASTICSEARCH_PASSWORD,
-					},
-					flushInterval: 1000,
-					'flush-bytes': 1000,
-				}
-				if (elasticConfig) {
-					Object.assign(esConfig, elasticConfig)
-				}
-
-				const esTransport = pinoElastic(esConfig)
-
-				transports.push(esTransport)
-			} else {
-				transports.push(
-					pino.destination({
-						minLength: 128,
-						sync: false,
-					})
-				)
-			}
+		// Validate log level - will throw if invalid
+		const logLevel = process.env.LOG_LEVEL || 'info'
+		isValidLogLevel(logLevel)
+
+		const transports = []
+		if (process.env.NODE_ENV !== 'local' && process.env.NODE_ENV !== 'test') {
+			// Validate Elasticsearch environment variables
+			validateElasticsearchEnv()
+
+			// Parse flush settings from environment with validation
+			const flushIntervalMs = parseIntEnv(
+				process.env.ES_FLUSH_INTERVAL_MS,
+				2000, // Default: 2 seconds
+				'ES_FLUSH_INTERVAL_MS'
+			)
+
+			const flushBytes = parseIntEnv(
+				process.env.ES_FLUSH_BYTES,
+				100 * 1024, // Default: 100KB
+				'ES_FLUSH_BYTES'
+			)
+
+			const maxRetries = parseIntEnv(
+				process.env.ES_MAX_RETRIES,
+				3, // Default: 3 retries
+				'ES_MAX_RETRIES'
+			)
 
-			pinoLogger = pino(
-				{
-					level: process.env.LOG_LEVEL,
-					timestamp: stdTimeFunctions.isoTime.bind(stdTimeFunctions),
+			const requestTimeout = parseIntEnv(
+				process.env.ES_REQUEST_TIMEOUT_MS,
+				30000, // Default: 30 seconds
+				'ES_REQUEST_TIMEOUT_MS'
+			)
+
+			// Safe to access after validation
+			const esConfig: ElasticConfig = {
+				index: process.env.SERVER_NICKNAME,
+				node: process.env.ELASTICSEARCH_NODE,
+				auth: {
+					username: process.env.ELASTICSEARCH_USERNAME as string,
+					password: process.env.ELASTICSEARCH_PASSWORD as string,
 				},
-				...transports
+				// Configurable flush settings
+				flushInterval: flushIntervalMs,
+				'flush-bytes': flushBytes,
+				// Retry configuration for connection resilience
+				maxRetries: maxRetries,
+				requestTimeout: requestTimeout,
+				// Automatically reconnect on connection faults
+				sniffOnConnectionFault: true,
+			}
+			if (elasticConfig) {
+				Object.assign(esConfig, elasticConfig)
+			}
+
+			// Create transport and store reference for cleanup
+			// Cast to PinoElasticOptions since our ElasticConfig includes ClientOptions properties
+			esTransport = pinoElastic(esConfig as PinoElasticOptions)
+
+			// Handle Elasticsearch connection errors
+			esTransport.on('error', (err) => {
+				console.error('[Logger] Elasticsearch transport error:', err.message)
+				console.error('[Logger] Logs may not be reaching Kibana. Check Elasticsearch connection.')
+			})
+
+			// Handle insert errors (document indexing failures)
+			esTransport.on('insertError', (err) => {
+				console.error('[Logger] Elasticsearch insert error:', err.message)
+				console.error('[Logger] Some logs failed to index to Elasticsearch.')
+			})
+
+			// Log successful connection (for debugging)
+			esTransport.on('insert', () => {
+				// Uncomment for debugging: console.log(`[Logger] Successfully inserted log entries`)
+			})
+
+			// Register shutdown handlers to flush logs on process exit
+			registerShutdownHandlers()
+
+			transports.push(esTransport)
+		} else {
+			transports.push(
+				pino.destination({
+					minLength: 1024,
+					sync: true,
+				})
 			)
 		}
+
+		pinoLogger = pino(
+			{
+				level: logLevel,
+				timestamp: stdTimeFunctions.isoTime.bind(stdTimeFunctions),
+			},
+			...transports
+		)
 	}
 	return pinoLogger
 }
@@ -103,8 +283,7 @@ class Logger {
 		if (process.env.NODE_ENV === 'local' || process.env.NODE_ENV === 'test') {
 			detail = args
 		} else {
-			//@ts-ignore
-			detail = JSON.stringify(...args)
+			detail = JSON.stringify(args)
 		}
 		this._logger[logLevel]({
 			component: this._name,

package/src/types/logger.ts

@@ -46,4 +46,21 @@ export interface ElasticConfig {
 	 * The interval (in milliseconds) at which logs are flushed to Elasticsearch.
 	 */
 	flushInterval?: number
+	/**
+	 * The number of bytes to buffer before flushing to Elasticsearch.
+	 */
+	'flush-bytes'?: number
+	/**
+	 * Maximum number of retries for failed Elasticsearch requests.
+	 */
+	maxRetries?: number
+	/**
+	 * Request timeout in milliseconds before considering a request failed.
+	 */
+	requestTimeout?: number
+	/**
+	 * Whether to sniff for additional Elasticsearch nodes on connection fault.
+	 * Enables automatic reconnection when a node fails.
+	 */
+	sniffOnConnectionFault?: boolean
 }