@mhmdhammoud/meritt-utils 1.5.8 → 1.6.0

package/dist/__tests__/logger.test.js

@@ -94,7 +94,7 @@ describe('route and format logs', () => {
         key0: 'val0',
         key1: 'val1',
     };
-    test('log info with structured context (single object flattened)', () => {
+    test('log info with structured context (single object in context field)', () => {
         //@ts-ignore
         jest.spyOn(pino_1.pino, 'destination').mockReturnValue(PINO_DESTINATION);
         //@ts-ignore
@@ -105,11 +105,10 @@ describe('route and format logs', () => {
             component: LOGGER_NAME,
             code: LOG_EVENT.code,
             msg: LOG_EVENT.msg,
-            key0: 'val0',
-            key1: 'val1',
+            context: { key0: 'val0', key1: 'val1' },
         }));
     });
-    test('remap reserved elastic field names and reduce objects to scalars', () => {
+    test('remap reserved elastic field names in context', () => {
         //@ts-ignore
         jest.spyOn(pino_1.pino, 'destination').mockReturnValue(PINO_DESTINATION);
         //@ts-ignore
@@ -122,10 +121,12 @@ describe('route and format logs', () => {
                 _index: 'bad-index',
             },
         });
-        // Top-level fields use scalars only (avoids ES document_parsing_exception)
+        // Context holds sanitized structure; reserved names remapped recursively
         expect(PINO.info).toHaveBeenCalledWith(expect.objectContaining({
-            mongo_id: 'abc123',
-            nested: 'nested-1',
+            context: expect.objectContaining({
+                mongo_id: 'abc123',
+                nested: { mongo_id: 'nested-1', es_index: 'bad-index' },
+            }),
         }));
         expect(PINO.info).not.toHaveBeenCalledWith(expect.objectContaining({
             _id: expect.anything(),

package/dist/lib/elastic-transport.js

@@ -33,24 +33,12 @@ function getIndexName(index, time) {
     }
     return index.replace('%{DATE}', time.substring(0, 10));
 }
-function initializeBulkHandler(opts, client, splitter) {
+function initializeBulkHandler(opts, client, splitter, onFatalError) {
     var _a, _b, _c, _d, _e, _f, _g;
     const esVersion = Number((_b = (_a = opts.esVersion) !== null && _a !== void 0 ? _a : opts['es-version']) !== null && _b !== void 0 ? _b : 7);
     const index = (_c = opts.index) !== null && _c !== void 0 ? _c : 'pino';
     const buildIndexName = typeof index === 'function' ? index : null;
     const opType = esVersion >= 7 ? undefined : undefined;
-    // CRITICAL FIX (issue #140): When bulk helper destroys stream after retries exhausted,
-    // we must BOTH resurrect the pool AND reinitialize the bulk handler so logging continues.
-    // connectionPool.resurrect exists at runtime (elastic-transport) but may not be in types
-    const pool = client.connectionPool;
-    const splitterWithDestroy = splitter;
-    splitterWithDestroy.destroy = function () {
-        if (typeof pool.resurrect === 'function') {
-            pool.resurrect({ name: 'elasticsearch-js' });
-        }
-        // Reinitialize bulk handler - without this, logging stops permanently until restart
-        initializeBulkHandler(opts, client, splitter);
-    };
     const indexName = (time = new Date().toISOString()) => buildIndexName ? buildIndexName(time) : getIndexName(index, time);
     const bulkInsert = client.helpers.bulk({
         datasource: splitter,
@@ -77,7 +65,10 @@ function initializeBulkHandler(opts, client, splitter) {
             splitter.emit('insertError', error);
         },
     });
-    bulkInsert.then((stats) => splitter.emit('insert', stats), (err) => splitter.emit('error', err));
+    bulkInsert.then((stats) => splitter.emit('insert', stats), (err) => {
+        splitter.emit('error', err);
+        onFatalError(err);
+    });
 }
 const createElasticTransport = (opts = {}) => {
     const splitter = split(function (line) {
@@ -128,10 +119,56 @@ const createElasticTransport = (opts = {}) => {
         clientOpts.ConnectionPool = opts.ConnectionPool;
     }
     const client = new elasticsearch_1.Client(clientOpts);
+    // CRITICAL FIX (pino-elasticsearch issues #140/#72): after retries are
+    // exhausted the bulk helper can stop consuming the stream while the process
+    // stays alive. Keep exactly one helper active and replace it after fatal
+    // helper failures instead of waiting for a server restart.
+    let isBulkHandlerActive = false;
+    let isRestartScheduled = false;
+    let isTransportClosed = false;
+    const pool = client.connectionPool;
+    const splitterWithDestroy = splitter;
+    const originalDestroy = splitterWithDestroy.destroy.bind(splitterWithDestroy);
+    const startBulkHandler = () => {
+        if (isTransportClosed || isBulkHandlerActive) {
+            return;
+        }
+        isBulkHandlerActive = true;
+        initializeBulkHandler(opts, client, splitter, () => {
+            isBulkHandlerActive = false;
+            scheduleBulkHandlerRestart();
+        });
+    };
+    const scheduleBulkHandlerRestart = () => {
+        var _a, _b, _c;
+        if (isTransportClosed || isRestartScheduled) {
+            return;
+        }
+        isRestartScheduled = true;
+        if (typeof pool.resurrect === 'function') {
+            pool.resurrect({ name: 'elasticsearch-js' });
+        }
+        const retryDelayMs = Math.min(Number((_b = (_a = opts.flushInterval) !== null && _a !== void 0 ? _a : opts['flush-interval']) !== null && _b !== void 0 ? _b : 3000), 5000);
+        const timer = setTimeout(() => {
+            isRestartScheduled = false;
+            startBulkHandler();
+        }, retryDelayMs);
+        (_c = timer.unref) === null || _c === void 0 ? void 0 : _c.call(timer);
+    };
+    splitterWithDestroy.destroy = function (err) {
+        if (err && !isTransportClosed) {
+            scheduleBulkHandlerRestart();
+            return;
+        }
+        isTransportClosed = true;
+        originalDestroy(err);
+    };
     client.diagnostic.on('resurrect', () => {
-        initializeBulkHandler(opts, client, splitter);
+        if (!isBulkHandlerActive) {
+            scheduleBulkHandlerRestart();
+        }
     });
-    initializeBulkHandler(opts, client, splitter);
+    startBulkHandler();
     return splitter;
 };
 exports.createElasticTransport = createElasticTransport;

package/dist/lib/logger.js

@@ -77,37 +77,6 @@ const isPlainObject = (v) => v !== null &&
     !Array.isArray(v) &&
     !(v instanceof Error) &&
     !(v instanceof Date);
-/**
- * Reduces object values to scalars for top-level ES fields.
- * ES text/keyword fields reject nested objects; use _id or truncated JSON.
- */
-const toScalarForTopLevel = (value) => {
-    if (value === null || value === undefined)
-        return null;
-    if (typeof value === 'string' ||
-        typeof value === 'number' ||
-        typeof value === 'boolean')
-        return value;
-    if (value instanceof Date)
-        return value.toISOString();
-    if (isObjectIdLike(value))
-        return value.toHexString();
-    if (value instanceof Error)
-        return value.message;
-    if (isPlainObject(value) && '_id' in value) {
-        const id = value._id;
-        if (id != null) {
-            if (typeof id === 'string')
-                return id;
-            if (isObjectIdLike(id))
-                return id.toHexString();
-            if (typeof id === 'object' && id !== null && 'toString' in id)
-                return String(id.toString());
-        }
-    }
-    const str = typeof value === 'object' ? JSON.stringify(value) : String(value);
-    return str.length > 200 ? `${str.slice(0, 200)}...` : str;
-};
 /**
  * Recursively sanitize log values for Elasticsearch safety.
  * - Remaps reserved key names (e.g. `_id` -> `mongo_id`)
@@ -412,17 +381,15 @@ class Logger {
         if (trace) {
             ecs.trace_id = trace.traceId;
         }
-        // Structured context: flatten single plain object as top-level fields
-        // Use scalars only for top-level ES fields; text/keyword mappings reject nested objects
+        // Structured context: put in single 'context' field to avoid ES mapping conflicts.
+        // Flattening to top-level caused document_parsing_exception (object vs scalar type mismatches).
+        // Nesting in context keeps structure consistent and avoids per-field mapping conflicts.
+        let context;
         let detail;
         if (args.length === 1 &&
             isPlainObject(args[0]) &&
             Object.keys(args[0]).length > 0) {
-            const obj = args[0];
-            for (const [k, v] of Object.entries(obj)) {
-                const key = toSafeElasticFieldName(k);
-                ecs[key] = toScalarForTopLevel(v);
-            }
+            context = sanitizeForElastic(args[0]);
         }
         else {
             detail = isLocal ? args : JSON.stringify(sanitizeForElastic(args));
@@ -434,6 +401,9 @@ class Logger {
             code: event.code,
             msg: event.msg,
         };
+        if (context !== undefined) {
+            base.context = context;
+        }
         if (detail !== undefined) {
             base.detail = detail;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@mhmdhammoud/meritt-utils",
-	"version": "1.5.8",
+	"version": "1.6.0",
 	"description": "",
 	"main": "./dist/index.js",
 	"private": false,

package/src/__tests__/logger.test.ts

@@ -70,7 +70,7 @@ describe('route and format logs', () => {
 		key0: 'val0',
 		key1: 'val1',
 	}
-	test('log info with structured context (single object flattened)', () => {
+	test('log info with structured context (single object in context field)', () => {
 		//@ts-ignore
 		jest.spyOn(pino, 'destination').mockReturnValue(PINO_DESTINATION)
 		//@ts-ignore
@@ -84,13 +84,12 @@ describe('route and format logs', () => {
 				component: LOGGER_NAME,
 				code: LOG_EVENT.code,
 				msg: LOG_EVENT.msg,
-				key0: 'val0',
-				key1: 'val1',
+				context: { key0: 'val0', key1: 'val1' },
 			})
 		)
 	})
 
-	test('remap reserved elastic field names and reduce objects to scalars', () => {
+	test('remap reserved elastic field names in context', () => {
 		//@ts-ignore
 		jest.spyOn(pino, 'destination').mockReturnValue(PINO_DESTINATION)
 		//@ts-ignore
@@ -105,11 +104,13 @@ describe('route and format logs', () => {
 			},
 		})
 
-		// Top-level fields use scalars only (avoids ES document_parsing_exception)
+		// Context holds sanitized structure; reserved names remapped recursively
 		expect(PINO.info).toHaveBeenCalledWith(
 			expect.objectContaining({
-				mongo_id: 'abc123',
-				nested: 'nested-1',
+				context: expect.objectContaining({
+					mongo_id: 'abc123',
+					nested: { mongo_id: 'nested-1', es_index: 'bad-index' },
+				}),
 			})
 		)
 		expect(PINO.info).not.toHaveBeenCalledWith(

package/src/lib/elastic-transport.ts

@@ -76,30 +76,14 @@ function getIndexName(
 function initializeBulkHandler(
 	opts: ElasticTransportOptions,
 	client: Client,
-	splitter: NodeJS.ReadWriteStream
+	splitter: NodeJS.ReadWriteStream,
+	onFatalError: (err: Error) => void
 ): void {
 	const esVersion = Number(opts.esVersion ?? opts['es-version'] ?? 7)
 	const index = opts.index ?? 'pino'
 	const buildIndexName = typeof index === 'function' ? index : null
 	const opType = esVersion >= 7 ? undefined : undefined
 
-	// CRITICAL FIX (issue #140): When bulk helper destroys stream after retries exhausted,
-	// we must BOTH resurrect the pool AND reinitialize the bulk handler so logging continues.
-	// connectionPool.resurrect exists at runtime (elastic-transport) but may not be in types
-	const pool = client.connectionPool as {
-		resurrect?: (opts: { name: string }) => void
-	}
-	const splitterWithDestroy = splitter as NodeJS.ReadWriteStream & {
-		destroy: (err?: Error) => void
-	}
-	splitterWithDestroy.destroy = function () {
-		if (typeof pool.resurrect === 'function') {
-			pool.resurrect({ name: 'elasticsearch-js' })
-		}
-		// Reinitialize bulk handler - without this, logging stops permanently until restart
-		initializeBulkHandler(opts, client, splitter)
-	}
-
 	const indexName = (time = new Date().toISOString()) =>
 		buildIndexName ? buildIndexName(time) : getIndexName(index as string, time)
 
@@ -132,7 +116,10 @@ function initializeBulkHandler(
 
 	bulkInsert.then(
 		(stats) => splitter.emit('insert', stats),
-		(err) => splitter.emit('error', err)
+		(err) => {
+			splitter.emit('error', err)
+			onFatalError(err)
+		}
 	)
 }
 
@@ -193,11 +180,70 @@ export const createElasticTransport = (
 
 	const client = new Client(clientOpts)
 
+	// CRITICAL FIX (pino-elasticsearch issues #140/#72): after retries are
+	// exhausted the bulk helper can stop consuming the stream while the process
+	// stays alive. Keep exactly one helper active and replace it after fatal
+	// helper failures instead of waiting for a server restart.
+	let isBulkHandlerActive = false
+	let isRestartScheduled = false
+	let isTransportClosed = false
+
+	const pool = client.connectionPool as {
+		resurrect?: (opts: { name: string }) => void
+	}
+	const splitterWithDestroy = splitter as NodeJS.ReadWriteStream & {
+		destroy: (err?: Error) => void
+	}
+	const originalDestroy = splitterWithDestroy.destroy.bind(splitterWithDestroy)
+
+	const startBulkHandler = () => {
+		if (isTransportClosed || isBulkHandlerActive) {
+			return
+		}
+		isBulkHandlerActive = true
+		initializeBulkHandler(opts, client, splitter, () => {
+			isBulkHandlerActive = false
+			scheduleBulkHandlerRestart()
+		})
+	}
+
+	const scheduleBulkHandlerRestart = () => {
+		if (isTransportClosed || isRestartScheduled) {
+			return
+		}
+		isRestartScheduled = true
+
+		if (typeof pool.resurrect === 'function') {
+			pool.resurrect({ name: 'elasticsearch-js' })
+		}
+
+		const retryDelayMs = Math.min(
+			Number(opts.flushInterval ?? opts['flush-interval'] ?? 3000),
+			5000
+		)
+		const timer = setTimeout(() => {
+			isRestartScheduled = false
+			startBulkHandler()
+		}, retryDelayMs)
+		timer.unref?.()
+	}
+
+	splitterWithDestroy.destroy = function (err?: Error) {
+		if (err && !isTransportClosed) {
+			scheduleBulkHandlerRestart()
+			return
+		}
+		isTransportClosed = true
+		originalDestroy(err)
+	}
+
 	client.diagnostic.on('resurrect', () => {
-		initializeBulkHandler(opts, client, splitter)
+		if (!isBulkHandlerActive) {
+			scheduleBulkHandlerRestart()
+		}
 	})
 
-	initializeBulkHandler(opts, client, splitter)
+	startBulkHandler()
 
 	return splitter
 }

package/src/lib/logger.ts

@@ -52,36 +52,6 @@ const isPlainObject = (v: unknown): v is Record<string, unknown> =>
 	!(v instanceof Error) &&
 	!(v instanceof Date)
 
-/**
- * Reduces object values to scalars for top-level ES fields.
- * ES text/keyword fields reject nested objects; use _id or truncated JSON.
- */
-const toScalarForTopLevel = (
-	value: unknown
-): string | number | boolean | null => {
-	if (value === null || value === undefined) return null
-	if (
-		typeof value === 'string' ||
-		typeof value === 'number' ||
-		typeof value === 'boolean'
-	)
-		return value
-	if (value instanceof Date) return value.toISOString()
-	if (isObjectIdLike(value)) return value.toHexString()
-	if (value instanceof Error) return value.message
-	if (isPlainObject(value) && '_id' in value) {
-		const id = (value as { _id?: unknown })._id
-		if (id != null) {
-			if (typeof id === 'string') return id
-			if (isObjectIdLike(id)) return id.toHexString()
-			if (typeof id === 'object' && id !== null && 'toString' in id)
-				return String((id as { toString: () => string }).toString())
-		}
-	}
-	const str = typeof value === 'object' ? JSON.stringify(value) : String(value)
-	return str.length > 200 ? `${str.slice(0, 200)}...` : str
-}
-
 /**
  * Recursively sanitize log values for Elasticsearch safety.
  * - Remaps reserved key names (e.g. `_id` -> `mongo_id`)
@@ -491,19 +461,17 @@ class Logger {
 			ecs.trace_id = trace.traceId
 		}
 
-		// Structured context: flatten single plain object as top-level fields
-		// Use scalars only for top-level ES fields; text/keyword mappings reject nested objects
+		// Structured context: put in single 'context' field to avoid ES mapping conflicts.
+		// Flattening to top-level caused document_parsing_exception (object vs scalar type mismatches).
+		// Nesting in context keeps structure consistent and avoids per-field mapping conflicts.
+		let context: Record<string, unknown> | undefined
 		let detail: unknown
 		if (
 			args.length === 1 &&
 			isPlainObject(args[0]) &&
 			Object.keys(args[0]).length > 0
 		) {
-			const obj = args[0]
-			for (const [k, v] of Object.entries(obj)) {
-				const key = toSafeElasticFieldName(k)
-				ecs[key] = toScalarForTopLevel(v)
-			}
+			context = sanitizeForElastic(args[0]) as Record<string, unknown>
 		} else {
 			detail = isLocal ? args : JSON.stringify(sanitizeForElastic(args))
 		}
@@ -515,6 +483,9 @@ class Logger {
 			code: event.code,
 			msg: event.msg,
 		}
+		if (context !== undefined) {
+			base.context = context
+		}
 		if (detail !== undefined) {
 			base.detail = detail
 		}