@mhmdhammoud/meritt-utils 1.5.1 → 1.5.3

package/dist/lib/logger.js

@@ -15,63 +15,208 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isValidLogLevel = void 0;
+exports.isValidLogLevel = isValidLogLevel;
 const pino_1 = require("pino");
 const dotenv = __importStar(require("dotenv"));
-const pino_elasticsearch_1 = __importDefault(require("pino-elasticsearch"));
+const elastic_transport_1 = require("./elastic-transport");
 dotenv.config();
 /**
  * Pino logger backend - singleton
  */
 let pinoLogger;
+/**
+ * Elasticsearch transport instance - kept for cleanup
+ */
+let esTransport = null;
+/**
+ * Flag to track if shutdown handlers are registered
+ */
+let shutdownHandlersRegistered = false;
+/**
+ * Validates required Elasticsearch environment variables.
+ * @throws Error if required environment variables are missing.
+ */
+function validateElasticsearchEnv() {
+    const required = [
+        'ELASTICSEARCH_NODE',
+        'ELASTICSEARCH_USERNAME',
+        'ELASTICSEARCH_PASSWORD',
+        'SERVER_NICKNAME',
+    ];
+    const missing = required.filter((key) => !process.env[key]);
+    if (missing.length > 0) {
+        throw new Error(`Missing required Elasticsearch environment variables: ${missing.join(', ')}`);
+    }
+}
+/**
+ * Safely parses an integer from an environment variable.
+ * @param envValue - The environment variable value to parse.
+ * @param defaultValue - The default value to use if parsing fails.
+ * @param varName - The name of the environment variable (for error messages).
+ * @returns The parsed integer or the default value.
+ * @throws Error if the value is not a valid number.
+ */
+function parseIntEnv(envValue, defaultValue, varName) {
+    if (!envValue) {
+        return defaultValue;
+    }
+    const parsed = parseInt(envValue, 10);
+    if (isNaN(parsed)) {
+        throw new Error(`Invalid value for ${varName}: "${envValue}" is not a valid number. Using default: ${defaultValue}`);
+    }
+    if (parsed < 0) {
+        throw new Error(`Invalid value for ${varName}: "${envValue}" must be a positive number. Using default: ${defaultValue}`);
+    }
+    return parsed;
+}
+/**
+ * Registers shutdown handlers to flush logs before process exits.
+ */
+function registerShutdownHandlers() {
+    if (shutdownHandlersRegistered) {
+        return;
+    }
+    let isShuttingDown = false;
+    const flushAndExit = async (signal, exitCode = 0) => {
+        // Prevent multiple shutdown attempts
+        if (isShuttingDown) {
+            return;
+        }
+        isShuttingDown = true;
+        if (esTransport) {
+            try {
+                // Flush any pending logs
+                await new Promise((resolve) => {
+                    const timeout = setTimeout(() => {
+                        console.error(`[Logger] Flush timeout after 5s on ${signal}`);
+                        resolve();
+                    }, 5000); // 5 second timeout
+                    // Register listener BEFORE calling end() to avoid race condition
+                    esTransport === null || esTransport === void 0 ? void 0 : esTransport.once('finish', () => {
+                        clearTimeout(timeout);
+                        resolve();
+                    });
+                    // Now trigger the flush
+                    esTransport === null || esTransport === void 0 ? void 0 : esTransport.end();
+                });
+            }
+            catch (error) {
+                console.error(`[Logger] Error flushing logs on ${signal}:`, error);
+            }
+        }
+        process.exit(exitCode);
+    };
+    process.on('SIGTERM', () => {
+        flushAndExit('SIGTERM', 0).catch((err) => {
+            console.error('[Logger] Flush and exit failed:', err);
+            process.exit(1);
+        });
+    });
+    process.on('SIGINT', () => {
+        flushAndExit('SIGINT', 130).catch((err) => {
+            console.error('[Logger] Flush and exit failed:', err);
+            process.exit(1);
+        });
+    });
+    process.on('beforeExit', () => {
+        if (esTransport && !isShuttingDown) {
+            esTransport.end();
+        }
+    });
+    shutdownHandlersRegistered = true;
+}
 /**
  * Creates a Pino logger instance with specified Elasticsearch configuration.
  * @param elasticConfig - Optional Elasticsearch configuration.
  * @returns The Pino logger instance.
+ * @throws Error if LOG_LEVEL is invalid or required environment variables are missing.
  */
 function getLogger(elasticConfig) {
     if (!pinoLogger) {
-        if (isValidLogLevel(process.env.LOG_LEVEL)) {
-            const transports = [];
-            if (process.env.NODE_ENV !== 'local' && process.env.NODE_ENV !== 'test') {
-                const esConfig = {
-                    index: process.env.SERVER_NICKNAME,
-                    node: process.env.ELASTICSEARCH_NODE,
-                    auth: {
-                        username: process.env.ELASTICSEARCH_USERNAME,
-                        password: process.env.ELASTICSEARCH_PASSWORD,
-                    },
-                    flushInterval: 1000,
-                    'flush-bytes': 1000,
-                };
-                if (elasticConfig) {
-                    Object.assign(esConfig, elasticConfig);
-                }
-                const esTransport = (0, pino_elasticsearch_1.default)(esConfig);
-                transports.push(esTransport);
-            }
-            else {
-                transports.push(pino_1.pino.destination({
-                    minLength: 128,
-                    sync: false,
-                }));
+        // Validate log level - will throw if invalid
+        const logLevel = process.env.LOG_LEVEL || 'info';
+        isValidLogLevel(logLevel);
+        const transports = [];
+        if (process.env.NODE_ENV !== 'local' && process.env.NODE_ENV !== 'test') {
+            // Validate Elasticsearch environment variables
+            validateElasticsearchEnv();
+            // Parse flush settings from environment with validation
+            const flushIntervalMs = parseIntEnv(process.env.ES_FLUSH_INTERVAL_MS, 2000, // Default: 2 seconds
+            'ES_FLUSH_INTERVAL_MS');
+            const flushBytes = parseIntEnv(process.env.ES_FLUSH_BYTES, 100 * 1024, // Default: 100KB
+            'ES_FLUSH_BYTES');
+            const maxRetries = parseIntEnv(process.env.ES_MAX_RETRIES, 3, // Default: 3 retries
+            'ES_MAX_RETRIES');
+            const requestTimeout = parseIntEnv(process.env.ES_REQUEST_TIMEOUT_MS, 30000, // Default: 30 seconds
+            'ES_REQUEST_TIMEOUT_MS');
+            // Safe to access after validation
+            const esConfig = {
+                index: process.env.SERVER_NICKNAME,
+                node: process.env.ELASTICSEARCH_NODE,
+                auth: {
+                    username: process.env.ELASTICSEARCH_USERNAME,
+                    password: process.env.ELASTICSEARCH_PASSWORD,
+                },
+                // Configurable flush settings
+                flushInterval: flushIntervalMs,
+                'flush-bytes': flushBytes,
+                // Retry configuration for connection resilience
+                maxRetries: maxRetries,
+                requestTimeout: requestTimeout,
+                // Automatically reconnect on connection faults
+                sniffOnConnectionFault: true,
+            };
+            if (elasticConfig) {
+                Object.assign(esConfig, elasticConfig);
             }
-            pinoLogger = (0, pino_1.pino)({
-                level: process.env.LOG_LEVEL,
-                timestamp: pino_1.stdTimeFunctions.isoTime.bind(pino_1.stdTimeFunctions),
-            }, ...transports);
+            // Create transport with connection lifecycle fix (pino-elasticsearch #140)
+            esTransport = (0, elastic_transport_1.createElasticTransport)(esConfig);
+            // Handle Elasticsearch connection errors
+            esTransport.on('error', (err) => {
+                console.error('[Logger] Elasticsearch transport error:', err.message);
+                console.error('[Logger] Logs may not be reaching Kibana. Check Elasticsearch connection.');
+            });
+            // Handle insert errors (document indexing failures)
+            esTransport.on('insertError', (err) => {
+                console.error('[Logger] Elasticsearch insert error:', err.message);
+                console.error('[Logger] Some logs failed to index to Elasticsearch.');
+            });
+            // Log successful connection (for debugging)
+            esTransport.on('insert', () => {
+                // Uncomment for debugging: console.log(`[Logger] Successfully inserted log entries`)
+            });
+            // Register shutdown handlers to flush logs on process exit
+            registerShutdownHandlers();
+            transports.push(esTransport);
         }
+        else {
+            transports.push(pino_1.pino.destination({
+                minLength: 1024,
+                sync: true,
+            }));
+        }
+        pinoLogger = (0, pino_1.pino)({
+            level: logLevel,
+            timestamp: pino_1.stdTimeFunctions.isoTime.bind(pino_1.stdTimeFunctions),
+        }, ...transports);
     }
     return pinoLogger;
 }
@@ -86,7 +231,6 @@ function isValidLogLevel(level) {
     }
     return true;
 }
-exports.isValidLogLevel = isValidLogLevel;
 /**
  * Logger Wrapper.
  * Wraps a Pino logger instance and provides logging methods.
@@ -102,8 +246,7 @@ class Logger {
             detail = args;
         }
         else {
-            //@ts-ignore
-            detail = JSON.stringify(...args);
+            detail = JSON.stringify(args);
         }
         this._logger[logLevel]({
             component: this._name,

package/dist/lib/pdf.d.ts

@@ -9,7 +9,7 @@ declare class Pdf {
      * // => 'Hello-World'
      * ```
      * */
-    getBase64Images: (url: string) => string;
+    getBase64Images: (_url: string) => string;
 }
 declare const pdf: Pdf;
 export default pdf;

package/dist/lib/pdf.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+// import { AxiosInstance } from '../utilities'
 /*
      Author : Mustafa Halabi https://github.com/mustafahalabi
      Date : 2023-06-24
@@ -18,7 +19,7 @@ class Pdf {
          * // => 'Hello-World'
          * ```
          * */
-        this.getBase64Images = (url) => {
+        this.getBase64Images = (_url) => {
             return '';
         };
     }

package/dist/types/logger.d.ts

@@ -44,4 +44,21 @@ export interface ElasticConfig {
      * The interval (in milliseconds) at which logs are flushed to Elasticsearch.
      */
     flushInterval?: number;
+    /**
+     * The number of bytes to buffer before flushing to Elasticsearch.
+     */
+    'flush-bytes'?: number;
+    /**
+     * Maximum number of retries for failed Elasticsearch requests.
+     */
+    maxRetries?: number;
+    /**
+     * Request timeout in milliseconds before considering a request failed.
+     */
+    requestTimeout?: number;
+    /**
+     * Whether to sniff for additional Elasticsearch nodes on connection fault.
+     * Enables automatic reconnection when a node fails.
+     */
+    sniffOnConnectionFault?: boolean;
 }

package/eslint.config.mjs

@@ -0,0 +1,64 @@
+// @ts-check
+import eslint from '@eslint/js'
+import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended'
+import globals from 'globals'
+import tseslint from 'typescript-eslint'
+
+export default tseslint.config(
+	{
+		ignores: [
+			'eslint.config.mjs',
+			'dist/**',
+			'node_modules/**',
+			'coverage/**',
+			'*.d.ts',
+		],
+	},
+	eslint.configs.recommended,
+	...tseslint.configs.recommendedTypeChecked,
+	eslintPluginPrettierRecommended,
+	{
+		languageOptions: {
+			globals: {
+				...globals.node,
+			},
+			sourceType: 'module',
+			parserOptions: {
+				projectService: true,
+				tsconfigRootDir: import.meta.dirname,
+			},
+		},
+	},
+	{
+		rules: {
+			'@typescript-eslint/no-explicit-any': 'off',
+			'@typescript-eslint/no-floating-promises': 'warn',
+			'@typescript-eslint/no-unsafe-argument': 'off',
+			'@typescript-eslint/no-unsafe-assignment': 'off',
+			'@typescript-eslint/no-unsafe-return': 'off',
+			'@typescript-eslint/no-unsafe-member-access': 'off',
+			'@typescript-eslint/no-unsafe-call': 'off',
+			'@typescript-eslint/ban-ts-comment': 'off',
+			'@typescript-eslint/require-await': 'off',
+			'@typescript-eslint/await-thenable': 'off',
+			'@typescript-eslint/no-unsafe-enum-comparison': 'off',
+			'@typescript-eslint/no-base-to-string': 'off',
+			'@typescript-eslint/no-redundant-type-constituents': 'off',
+			'@typescript-eslint/no-unused-vars': [
+				'error',
+				{
+					argsIgnorePattern: '^_',
+					varsIgnorePattern: '^_',
+				},
+			],
+			'no-console': 'error',
+			'prettier/prettier': ['error', { semi: false }],
+		},
+	},
+	{
+		files: ['src/lib/logger.ts'],
+		rules: {
+			'no-console': 'off',
+		},
+	}
+)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@mhmdhammoud/meritt-utils",
-	"version": "1.5.1",
+	"version": "1.5.3",
 	"description": "",
 	"main": "./dist/index.js",
 	"private": false,
@@ -25,25 +25,31 @@
 		"test:coverage": "jest --coverage"
 	},
 	"devDependencies": {
-		"@types/jest": "^27.5.2",
-		"@typescript-eslint/eslint-plugin": "^5.17.0",
-		"@typescript-eslint/parser": "^5.17.0",
-		"eslint": "^8.12.0",
-		"eslint-plugin-tsdoc": "^0.2.14",
-		"husky": "^8.0.0",
+		"@eslint/js": "^9.31.0",
+		"@types/jest": "^29.5.12",
+		"eslint": "^9.31.0",
+		"eslint-config-prettier": "^10.1.8",
+		"eslint-plugin-prettier": "^5.5.3",
+		"eslint-plugin-tsdoc": "^0.5.2",
+		"globals": "^16.0.0",
+		"husky": "^9.1.6",
 		"jest": "^29.7.0",
 		"pino-pretty": "^10.3.1",
+		"prettier": "^3.3.3",
 		"ts-jest": "^29.1.1",
-		"ts-node-dev": "^1.1.8",
-		"typescript": "^4.6.3"
+		"ts-node-dev": "^2.0.0",
+		"typescript": "^5.3.3",
+		"typescript-eslint": "^8.20.0"
 	},
 	"author": "Mhmdhammoud",
 	"license": "ISC",
 	"dependencies": {
+		"@elastic/elasticsearch": "^9.3.2",
 		"axios": "^1.4.0",
 		"dotenv": "^16.4.1",
 		"imagesloaded": "^5.0.0",
 		"pino": "^8.19.0",
-		"pino-elasticsearch": "^8.0.0"
+		"pino-elasticsearch": "^8.0.0",
+		"split2": "^4.2.0"
 	}
 }

package/src/__tests__/colorful.test.ts

@@ -1,4 +1,4 @@
-import {Colorful} from '../lib'
+import { Colorful } from '../lib'
 
 describe('Colorful Class', () => {
 	describe('rgbToHex method', () => {

package/src/__tests__/formatter.test.ts

@@ -1,4 +1,4 @@
-import {Formatter} from '../lib'
+import { Formatter } from '../lib'
 
 describe('Formatter', () => {
 	describe('toUpperFirst method', () => {

package/src/__tests__/logger.test.ts

@@ -1,5 +1,5 @@
-import Logger, {isValidLogLevel} from '../lib/logger'
-import {pino} from 'pino'
+import Logger, { isValidLogLevel } from '../lib/logger'
+import { pino } from 'pino'
 
 jest.mock('pino')
 

package/src/lib/elastic-transport.ts

@@ -0,0 +1,203 @@
+/**
+ * Elasticsearch transport for Pino with connection lifecycle resilience.
+ *
+ * Based on pino-elasticsearch with a fix for GitHub issue #140:
+ * When maxRetries are exceeded and Elasticsearch nodes are DEAD, the bulk helper
+ * destroys the splitter stream, causing logs to stop permanently until restart.
+ *
+ * This implementation overrides splitter.destroy to BOTH resurrect the connection
+ * pool AND reinitialize the bulk handler, so logging continues after ES recovers.
+ *
+ * @see https://github.com/pinojs/pino-elasticsearch/issues/140
+ * @see https://github.com/pinojs/pino-elasticsearch/issues/72
+ */
+
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const split = require('split2') as (
+	fn: (line: string) => unknown,
+	opts?: { autoDestroy?: boolean }
+) => NodeJS.ReadWriteStream
+import { Readable } from 'stream'
+import { Client } from '@elastic/elasticsearch'
+import type { ClientOptions } from '@elastic/elasticsearch'
+
+export interface ElasticTransportOptions extends Pick<
+	ClientOptions,
+	| 'node'
+	| 'auth'
+	| 'cloud'
+	| 'caFingerprint'
+	| 'Connection'
+	| 'ConnectionPool'
+	| 'maxRetries'
+	| 'requestTimeout'
+> {
+	sniffOnConnectionFault?: boolean
+	index?: string | ((logTime: string) => string)
+	flushBytes?: number
+	'flush-bytes'?: number
+	flushInterval?: number
+	'flush-interval'?: number
+	esVersion?: number
+	'es-version'?: number
+	rejectUnauthorized?: boolean
+	tls?: ClientOptions['tls']
+}
+
+interface LogDocument {
+	time?: string
+	'@timestamp'?: string
+	[k: string]: unknown
+}
+
+function setDateTimeString(value: unknown): string {
+	if (value !== null && typeof value === 'object' && 'time' in value) {
+		const t = (value as { time: unknown }).time
+		if (
+			(typeof t === 'string' && t.length > 0) ||
+			(typeof t === 'number' && t >= 0)
+		) {
+			return new Date(t).toISOString()
+		}
+	}
+	return new Date().toISOString()
+}
+
+function getIndexName(
+	index: string | ((logTime: string) => string),
+	time: string
+): string {
+	if (typeof index === 'function') {
+		return index(time)
+	}
+	return index.replace('%{DATE}', time.substring(0, 10))
+}
+
+function initializeBulkHandler(
+	opts: ElasticTransportOptions,
+	client: Client,
+	splitter: NodeJS.ReadWriteStream
+): void {
+	const esVersion = Number(opts.esVersion ?? opts['es-version'] ?? 7)
+	const index = opts.index ?? 'pino'
+	const buildIndexName = typeof index === 'function' ? index : null
+	const opType = esVersion >= 7 ? undefined : undefined
+
+	// CRITICAL FIX (issue #140): When bulk helper destroys stream after retries exhausted,
+	// we must BOTH resurrect the pool AND reinitialize the bulk handler so logging continues.
+	// connectionPool.resurrect exists at runtime (elastic-transport) but may not be in types
+	const pool = client.connectionPool as {
+		resurrect?: (opts: { name: string }) => void
+	}
+	const splitterWithDestroy = splitter as NodeJS.ReadWriteStream & {
+		destroy: (err?: Error) => void
+	}
+	splitterWithDestroy.destroy = function () {
+		if (typeof pool.resurrect === 'function') {
+			pool.resurrect({ name: 'elasticsearch-js' })
+		}
+		// Reinitialize bulk handler - without this, logging stops permanently until restart
+		initializeBulkHandler(opts, client, splitter)
+	}
+
+	const indexName = (time = new Date().toISOString()) =>
+		buildIndexName ? buildIndexName(time) : getIndexName(index as string, time)
+
+	const bulkInsert = client.helpers.bulk({
+		datasource: splitter as unknown as Readable,
+		flushBytes: opts.flushBytes ?? opts['flush-bytes'] ?? 1000,
+		flushInterval: opts.flushInterval ?? opts['flush-interval'] ?? 3000,
+		refreshOnCompletion: indexName(),
+		onDocument(doc: unknown) {
+			const d = doc as LogDocument
+			const date = d.time ?? d['@timestamp'] ?? new Date().toISOString()
+			if (opType === 'create') {
+				d['@timestamp'] = date
+			}
+			return {
+				index: {
+					_index: indexName(date),
+					op_type: opType,
+				},
+			}
+		},
+		onDrop(doc: unknown) {
+			const error = new Error('Dropped document') as Error & {
+				document: unknown
+			}
+			error.document = doc
+			splitter.emit('insertError', error)
+		},
+	})
+
+	bulkInsert.then(
+		(stats) => splitter.emit('insert', stats),
+		(err) => splitter.emit('error', err)
+	)
+}
+
+export const createElasticTransport = (
+	opts: ElasticTransportOptions = {}
+): NodeJS.ReadWriteStream => {
+	const splitter = split(
+		function (this: NodeJS.ReadWriteStream, line: string) {
+			let value: unknown
+
+			try {
+				value = JSON.parse(line) as unknown
+			} catch (error) {
+				this.emit('unknown', line, error)
+				return
+			}
+
+			if (typeof value === 'boolean') {
+				this.emit('unknown', line, 'Boolean value ignored')
+				return
+			}
+			if (value === null) {
+				this.emit('unknown', line, 'Null value ignored')
+				return
+			}
+			if (typeof value !== 'object') {
+				value = { data: value, time: setDateTimeString(value) }
+			} else {
+				const obj = value as Record<string, unknown>
+				if (obj['@timestamp'] === undefined) {
+					;(obj as LogDocument).time = setDateTimeString(obj)
+				}
+			}
+			return value
+		},
+		{ autoDestroy: true }
+	)
+
+	const clientOpts: ClientOptions = {
+		node: opts.node,
+		auth: opts.auth,
+		cloud: opts.cloud,
+		tls: { rejectUnauthorized: opts.rejectUnauthorized, ...opts.tls },
+		maxRetries: opts.maxRetries,
+		requestTimeout: opts.requestTimeout,
+		sniffOnConnectionFault: opts.sniffOnConnectionFault,
+	}
+
+	if (opts.caFingerprint) {
+		clientOpts.caFingerprint = opts.caFingerprint
+	}
+	if (opts.Connection) {
+		clientOpts.Connection = opts.Connection
+	}
+	if (opts.ConnectionPool) {
+		clientOpts.ConnectionPool = opts.ConnectionPool
+	}
+
+	const client = new Client(clientOpts)
+
+	client.diagnostic.on('resurrect', () => {
+		initializeBulkHandler(opts, client, splitter)
+	})
+
+	initializeBulkHandler(opts, client, splitter)
+
+	return splitter
+}

package/src/lib/imagefull.ts

@@ -28,7 +28,7 @@ class ImageFull {
 		return new Promise((resolve) => {
 			imagesLoaded(
 				document.querySelectorAll(selector),
-				{background: true},
+				{ background: true },
 				(event) => {
 					resolve(event)
 				}

package/src/lib/index.ts

@@ -1,6 +1,6 @@
-export {default as Crypto} from './cypto'
-export {default as Formatter} from './formatter'
-export {default as Pdf} from './formatter'
-export {default as Colorful} from './colorful'
-export {default as ImageFull} from './imagefull'
-export {default as Logger} from './logger'
+export { default as Crypto } from './cypto'
+export { default as Formatter } from './formatter'
+export { default as Pdf } from './formatter'
+export { default as Colorful } from './colorful'
+export { default as ImageFull } from './imagefull'
+export { default as Logger } from './logger'