@mhmdhammoud/meritt-utils 1.6.1 → 1.6.2

package/dist/__tests__/elastic-transport.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/elastic-transport.test.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const elastic_transport_1 = require("../lib/elastic-transport");
+const elasticsearch_1 = require("@elastic/elasticsearch");
+jest.mock('@elastic/elasticsearch', () => ({
+    Client: jest.fn(),
+}));
+const MockedClient = elasticsearch_1.Client;
+const writeLog = (transport, doc) => transport.write(`${JSON.stringify(doc)}\n`);
+const waitForEvent = (emitter, eventName) => new Promise((resolve) => {
+    emitter.once(eventName, (value) => resolve(value));
+});
+describe('elastic transport resilience', () => {
+    beforeEach(() => {
+        jest.useRealTimers();
+        MockedClient.mockReset();
+    });
+    test('reports bulk failures without emitting terminal stream errors', async () => {
+        const bulk = jest
+            .fn()
+            .mockRejectedValueOnce(new Error('ES unavailable'))
+            .mockResolvedValueOnce({ errors: false, items: [] });
+        MockedClient.mockImplementation(() => ({ bulk }));
+        const transport = (0, elastic_transport_1.createElasticTransport)({
+            index: 'logs',
+            flushBytes: 1,
+            flushInterval: 10000,
+            requestTimeout: 10,
+        });
+        const streamError = jest.fn();
+        transport.on('error', streamError);
+        const failed = waitForEvent(transport, 'bulkError');
+        writeLog(transport, { message: 'first' });
+        await failed;
+        const inserted = waitForEvent(transport, 'insert');
+        writeLog(transport, { message: 'second' });
+        await expect(inserted).resolves.toEqual({ successful: 1, failed: 0 });
+        expect(streamError).not.toHaveBeenCalled();
+        expect(bulk).toHaveBeenCalledTimes(2);
+    });
+    test('watchdog releases a stuck bulk request so later logs can flush', async () => {
+        jest.useFakeTimers();
+        const bulk = jest
+            .fn()
+            .mockReturnValueOnce(new Promise(() => undefined))
+            .mockResolvedValueOnce({ errors: false, items: [] });
+        MockedClient.mockImplementation(() => ({ bulk }));
+        const transport = (0, elastic_transport_1.createElasticTransport)({
+            index: 'logs',
+            flushBytes: 1,
+            flushInterval: 10000,
+            requestTimeout: 1,
+        });
+        const timedOut = waitForEvent(transport, 'bulkError');
+        writeLog(transport, { message: 'stuck' });
+        await jest.advanceTimersByTimeAsync(5001);
+        await expect(timedOut).resolves.toMatchObject({
+            message: 'Elasticsearch bulk request timed out after 5001ms',
+        });
+        const inserted = waitForEvent(transport, 'insert');
+        writeLog(transport, { message: 'recovered' });
+        await expect(inserted).resolves.toEqual({ successful: 1, failed: 0 });
+        expect(bulk).toHaveBeenCalledTimes(2);
+    });
+});

package/dist/lib/elastic-transport.d.ts

@@ -24,4 +24,8 @@ export interface ElasticTransportOptions extends Pick<ClientOptions, 'node' | 'a
     rejectUnauthorized?: boolean;
     tls?: ClientOptions['tls'];
 }
-export declare const createElasticTransport: (opts?: ElasticTransportOptions) => NodeJS.ReadWriteStream;
+export interface ElasticTransportStream extends NodeJS.ReadWriteStream {
+    flush: () => Promise<void>;
+    close: () => Promise<void>;
+}
+export declare const createElasticTransport: (opts?: ElasticTransportOptions) => ElasticTransportStream;

package/dist/lib/elastic-transport.js

@@ -17,6 +17,7 @@ exports.createElasticTransport = void 0;
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 const split = require('split2');
 const elasticsearch_1 = require("@elastic/elasticsearch");
+const createTimeoutError = (timeoutMs) => new Error(`Elasticsearch bulk request timed out after ${timeoutMs}ms`);
 function setDateTimeString(value) {
     if (value !== null && typeof value === 'object' && 'time' in value) {
         const t = value.time;
@@ -34,13 +35,15 @@ function getIndexName(index, time) {
     return index.replace('%{DATE}', time.substring(0, 10));
 }
 function createBulkSender(opts, client, splitter) {
-    var _a, _b, _c, _d, _e, _f, _g;
+    var _a, _b, _c, _d, _e, _f, _g, _h;
     const esVersion = Number((_b = (_a = opts.esVersion) !== null && _a !== void 0 ? _a : opts['es-version']) !== null && _b !== void 0 ? _b : 7);
     const index = (_c = opts.index) !== null && _c !== void 0 ? _c : 'pino';
     const buildIndexName = typeof index === 'function' ? index : null;
     const opType = esVersion >= 7 ? undefined : undefined;
     const flushBytes = (_e = (_d = opts.flushBytes) !== null && _d !== void 0 ? _d : opts['flush-bytes']) !== null && _e !== void 0 ? _e : 1000;
     const flushInterval = (_g = (_f = opts.flushInterval) !== null && _f !== void 0 ? _f : opts['flush-interval']) !== null && _g !== void 0 ? _g : 3000;
+    const requestTimeout = (_h = opts.requestTimeout) !== null && _h !== void 0 ? _h : 30000;
+    const bulkWatchdogTimeout = requestTimeout + 5000;
     let buffer = [];
     let bufferedBytes = 0;
     let timer;
@@ -100,6 +103,40 @@ function createBulkSender(opts, client, splitter) {
         error.cause = cause;
         splitter.emit('insertError', error);
     };
+    const emitBulkError = (err) => {
+        // Do not emit the standard stream "error" event for retryable bulk
+        // failures. Some stream consumers treat it as terminal, which can leave
+        // Pino writing into a poisoned stream while the process continues running.
+        splitter.emit('bulkError', err);
+    };
+    const bulkWithWatchdog = async (operations) => {
+        let timeout;
+        const bulkPromise = client.bulk({
+            operations,
+            refresh: false,
+            timeout: `${requestTimeout}ms`,
+        });
+        try {
+            return await Promise.race([
+                bulkPromise,
+                new Promise((_, reject) => {
+                    var _a;
+                    timeout = setTimeout(() => {
+                        reject(createTimeoutError(bulkWatchdogTimeout));
+                    }, bulkWatchdogTimeout);
+                    (_a = timeout.unref) === null || _a === void 0 ? void 0 : _a.call(timeout);
+                }),
+            ]);
+        }
+        finally {
+            if (timeout) {
+                clearTimeout(timeout);
+            }
+            // If the watchdog wins, the original request may reject later. Consume
+            // that rejection so a stale request cannot crash the app.
+            bulkPromise.catch(() => undefined);
+        }
+    };
     const flush = async () => {
         var _a, _b;
         if (isFlushing) {
@@ -116,12 +153,7 @@ function createBulkSender(opts, client, splitter) {
         bufferedBytes = 0;
         try {
             const operations = batch.flatMap(buildOperation);
-            const response = await client.bulk({
-                operations,
-                refresh: false,
-                timeout: opts.requestTimeout ? `${opts.requestTimeout}ms` : undefined,
-            });
-            const body = response;
+            const body = await bulkWithWatchdog(operations);
             if (body.errors && Array.isArray(body.items)) {
                 body.items.forEach((item, index) => {
                     const result = Object.values(item)[0];
@@ -132,11 +164,13 @@ function createBulkSender(opts, client, splitter) {
             }
             splitter.emit('insert', {
                 successful: batch.length,
-                failed: body.errors ? ((_b = (_a = body.items) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0) : 0,
+                failed: body.errors
+                    ? ((_b = (_a = body.items) === null || _a === void 0 ? void 0 : _a.filter((item) => { var _a; return (_a = Object.values(item)[0]) === null || _a === void 0 ? void 0 : _a.error; }).length) !== null && _b !== void 0 ? _b : 0)
+                    : 0,
             });
         }
         catch (err) {
-            splitter.emit('error', err);
+            emitBulkError(err);
             // Drop the failed batch instead of wedging the stream. The next log line
             // creates a fresh bulk request and can recover without a process restart.
             batch.forEach((doc) => emitDroppedDocument(doc, err));
@@ -226,11 +260,13 @@ const createElasticTransport = (opts = {}) => {
         void bulkSender.close();
     });
     const splitterWithDestroy = splitter;
+    splitterWithDestroy.flush = bulkSender.flush;
+    splitterWithDestroy.close = bulkSender.close;
     const originalDestroy = splitterWithDestroy.destroy.bind(splitterWithDestroy);
     splitterWithDestroy.destroy = function (err) {
         void bulkSender.close();
         originalDestroy(err);
     };
-    return splitter;
+    return splitterWithDestroy;
 };
 exports.createElasticTransport = createElasticTransport;

package/dist/lib/logger.js

@@ -219,8 +219,12 @@ function registerShutdownHandlers() {
                         clearTimeout(timeout);
                         resolve();
                     });
-                    // Now trigger the flush
-                    esTransport === null || esTransport === void 0 ? void 0 : esTransport.end();
+                    esTransport === null || esTransport === void 0 ? void 0 : esTransport.flush().catch((error) => {
+                        console.error(`[Logger] Error flushing logs on ${signal}:`, error);
+                    }).finally(() => {
+                        // Now trigger stream shutdown
+                        esTransport === null || esTransport === void 0 ? void 0 : esTransport.end();
+                    });
                 });
             }
             catch (error) {
@@ -295,7 +299,7 @@ function getLogger(elasticConfig) {
             // Create transport with connection lifecycle fix (pino-elasticsearch #140)
             esTransport = (0, elastic_transport_1.createElasticTransport)(esConfig);
             // Handle Elasticsearch connection errors
-            esTransport.on('error', (err) => {
+            esTransport.on('bulkError', (err) => {
                 console.error('[Logger] Elasticsearch transport error:', err.message);
                 console.error('[Logger] Logs may not be reaching Kibana. Check Elasticsearch connection.');
             });

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@mhmdhammoud/meritt-utils",
-	"version": "1.6.1",
+	"version": "1.6.2",
 	"description": "",
 	"main": "./dist/index.js",
 	"private": false,

package/src/__tests__/elastic-transport.test.ts

@@ -0,0 +1,94 @@
+import { createElasticTransport } from '../lib/elastic-transport'
+import { Client } from '@elastic/elasticsearch'
+import { EventEmitter } from 'events'
+
+jest.mock('@elastic/elasticsearch', () => ({
+	Client: jest.fn(),
+}))
+
+const MockedClient = Client as jest.Mock
+
+const writeLog = (
+	transport: NodeJS.WritableStream,
+	doc: Record<string, unknown>
+) => transport.write(`${JSON.stringify(doc)}\n`)
+
+const waitForEvent = <T = unknown>(
+	emitter: EventEmitter,
+	eventName: string
+): Promise<T> =>
+	new Promise((resolve) => {
+		emitter.once(eventName, (value) => resolve(value as T))
+	})
+
+describe('elastic transport resilience', () => {
+	beforeEach(() => {
+		jest.useRealTimers()
+		MockedClient.mockReset()
+	})
+
+	test('reports bulk failures without emitting terminal stream errors', async () => {
+		const bulk = jest
+			.fn()
+			.mockRejectedValueOnce(new Error('ES unavailable'))
+			.mockResolvedValueOnce({ errors: false, items: [] })
+		MockedClient.mockImplementation(() => ({ bulk }))
+
+		const transport = createElasticTransport({
+			index: 'logs',
+			flushBytes: 1,
+			flushInterval: 10000,
+			requestTimeout: 10,
+		})
+		const streamError = jest.fn()
+		transport.on('error', streamError)
+
+		const failed = waitForEvent<Error>(transport, 'bulkError')
+		writeLog(transport, { message: 'first' })
+		await failed
+
+		const inserted = waitForEvent<{ successful: number; failed: number }>(
+			transport,
+			'insert'
+		)
+		writeLog(transport, { message: 'second' })
+
+		await expect(inserted).resolves.toEqual({ successful: 1, failed: 0 })
+		expect(streamError).not.toHaveBeenCalled()
+		expect(bulk).toHaveBeenCalledTimes(2)
+	})
+
+	test('watchdog releases a stuck bulk request so later logs can flush', async () => {
+		jest.useFakeTimers()
+
+		const bulk = jest
+			.fn()
+			.mockReturnValueOnce(new Promise(() => undefined))
+			.mockResolvedValueOnce({ errors: false, items: [] })
+		MockedClient.mockImplementation(() => ({ bulk }))
+
+		const transport = createElasticTransport({
+			index: 'logs',
+			flushBytes: 1,
+			flushInterval: 10000,
+			requestTimeout: 1,
+		})
+
+		const timedOut = waitForEvent<Error>(transport, 'bulkError')
+		writeLog(transport, { message: 'stuck' })
+
+		await jest.advanceTimersByTimeAsync(5001)
+		await expect(timedOut).resolves.toMatchObject({
+			message: 'Elasticsearch bulk request timed out after 5001ms',
+		})
+
+		const inserted = waitForEvent<{ successful: number; failed: number }>(
+			transport,
+			'insert'
+		)
+		writeLog(transport, { message: 'recovered' })
+
+		await expect(inserted).resolves.toEqual({ successful: 1, failed: 0 })
+		expect(bulk).toHaveBeenCalledTimes(2)
+	})
+})

package/src/lib/elastic-transport.ts

@@ -49,6 +49,14 @@ interface LogDocument {
 	[k: string]: unknown
 }
 
+export interface ElasticTransportStream extends NodeJS.ReadWriteStream {
+	flush: () => Promise<void>
+	close: () => Promise<void>
+}
+
+const createTimeoutError = (timeoutMs: number): Error =>
+	new Error(`Elasticsearch bulk request timed out after ${timeoutMs}ms`)
+
 function setDateTimeString(value: unknown): string {
 	if (value !== null && typeof value === 'object' && 'time' in value) {
 		const t = (value as { time: unknown }).time
@@ -87,6 +95,8 @@ function createBulkSender(
 	const opType = esVersion >= 7 ? undefined : undefined
 	const flushBytes = opts.flushBytes ?? opts['flush-bytes'] ?? 1000
 	const flushInterval = opts.flushInterval ?? opts['flush-interval'] ?? 3000
+	const requestTimeout = opts.requestTimeout ?? 30000
+	const bulkWatchdogTimeout = requestTimeout + 5000
 
 	let buffer: unknown[] = []
 	let bufferedBytes = 0
@@ -154,6 +164,49 @@ function createBulkSender(
 		splitter.emit('insertError', error)
 	}
 
+	const emitBulkError = (err: unknown) => {
+		// Do not emit the standard stream "error" event for retryable bulk
+		// failures. Some stream consumers treat it as terminal, which can leave
+		// Pino writing into a poisoned stream while the process continues running.
+		splitter.emit('bulkError', err)
+	}
+
+	const bulkWithWatchdog = async (
+		operations: unknown[]
+	): Promise<{
+		errors?: boolean
+		items?: Array<Record<string, { error?: unknown }>>
+	}> => {
+		let timeout: NodeJS.Timeout | undefined
+		const bulkPromise = client.bulk({
+			operations,
+			refresh: false,
+			timeout: `${requestTimeout}ms`,
+		}) as Promise<{
+			errors?: boolean
+			items?: Array<Record<string, { error?: unknown }>>
+		}>
+
+		try {
+			return await Promise.race([
+				bulkPromise,
+				new Promise<never>((_, reject) => {
+					timeout = setTimeout(() => {
+						reject(createTimeoutError(bulkWatchdogTimeout))
+					}, bulkWatchdogTimeout)
+					timeout.unref?.()
+				}),
+			])
+		} finally {
+			if (timeout) {
+				clearTimeout(timeout)
+			}
+			// If the watchdog wins, the original request may reject later. Consume
+			// that rejection so a stale request cannot crash the app.
+			bulkPromise.catch(() => undefined)
+		}
+	}
+
 	const flush = async (): Promise<void> => {
 		if (isFlushing) {
 			flushAgain = true
@@ -171,16 +224,7 @@ function createBulkSender(
 
 		try {
 			const operations = batch.flatMap(buildOperation)
-			const response = await client.bulk({
-				operations,
-				refresh: false,
-				timeout: opts.requestTimeout ? `${opts.requestTimeout}ms` : undefined,
-			})
-
-			const body = response as {
-				errors?: boolean
-				items?: Array<Record<string, { error?: unknown }>>
-			}
+			const body = await bulkWithWatchdog(operations)
 			if (body.errors && Array.isArray(body.items)) {
 				body.items.forEach((item, index) => {
 					const result = Object.values(item)[0]
@@ -192,10 +236,13 @@ function createBulkSender(
 
 			splitter.emit('insert', {
 				successful: batch.length,
-				failed: body.errors ? (body.items?.length ?? 0) : 0,
+				failed: body.errors
+					? (body.items?.filter((item) => Object.values(item)[0]?.error)
+							.length ?? 0)
+					: 0,
 			})
 		} catch (err) {
-			splitter.emit('error', err)
+			emitBulkError(err)
 			// Drop the failed batch instead of wedging the stream. The next log line
 			// creates a fresh bulk request and can recover without a process restart.
 			batch.forEach((doc) => emitDroppedDocument(doc, err))
@@ -231,7 +278,7 @@ function createBulkSender(
 
 export const createElasticTransport = (
 	opts: ElasticTransportOptions = {}
-): NodeJS.ReadWriteStream => {
+): ElasticTransportStream => {
 	const splitter = split(
 		function (this: NodeJS.ReadWriteStream, line: string) {
 			let value: unknown
@@ -294,14 +341,16 @@ export const createElasticTransport = (
 		void bulkSender.close()
 	})
 
-	const splitterWithDestroy = splitter as NodeJS.ReadWriteStream & {
+	const splitterWithDestroy = splitter as ElasticTransportStream & {
 		destroy: (err?: Error) => void
 	}
+	splitterWithDestroy.flush = bulkSender.flush
+	splitterWithDestroy.close = bulkSender.close
 	const originalDestroy = splitterWithDestroy.destroy.bind(splitterWithDestroy)
 	splitterWithDestroy.destroy = function (err?: Error) {
 		void bulkSender.close()
 		originalDestroy(err)
 	}
 
-	return splitter
+	return splitterWithDestroy
 }

package/src/lib/logger.ts

@@ -1,7 +1,10 @@
 import { Logger as PinoLogger, pino, stdTimeFunctions } from 'pino'
 import * as dotenv from 'dotenv'
 import { hostname } from 'os'
-import { createElasticTransport } from './elastic-transport'
+import {
+	createElasticTransport,
+	ElasticTransportStream,
+} from './elastic-transport'
 import { getTraceContext } from './trace-store'
 import { LOG_LEVEL, LogEvent, ElasticConfig } from '../types'
 
@@ -140,7 +143,7 @@ let pinoLogger: PinoLogger
 /**
  * Elasticsearch transport instance - kept for cleanup
  */
-let esTransport: NodeJS.ReadWriteStream | null = null
+let esTransport: ElasticTransportStream | null = null
 
 /**
  * Flag to track if shutdown handlers are registered
@@ -233,8 +236,15 @@ function registerShutdownHandlers(): void {
 						resolve()
 					})
 
-					// Now trigger the flush
-					esTransport?.end()
+					esTransport
+						?.flush()
+						.catch((error) => {
+							console.error(`[Logger] Error flushing logs on ${signal}:`, error)
+						})
+						.finally(() => {
+							// Now trigger stream shutdown
+							esTransport?.end()
+						})
 				})
 			} catch (error) {
 				console.error(`[Logger] Error flushing logs on ${signal}:`, error)
@@ -333,7 +343,7 @@ function getLogger(elasticConfig?: ElasticConfig): PinoLogger {
 			esTransport = createElasticTransport(esConfig)
 
 			// Handle Elasticsearch connection errors
-			esTransport.on('error', (err: Error) => {
+			esTransport.on('bulkError', (err: Error) => {
 				console.error('[Logger] Elasticsearch transport error:', err.message)
 				console.error(
 					'[Logger] Logs may not be reaching Kibana. Check Elasticsearch connection.'