@mhmdhammoud/meritt-utils 1.6.0 → 1.6.2

package/dist/__tests__/elastic-transport.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/elastic-transport.test.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const elastic_transport_1 = require("../lib/elastic-transport");
+const elasticsearch_1 = require("@elastic/elasticsearch");
+jest.mock('@elastic/elasticsearch', () => ({
+    Client: jest.fn(),
+}));
+const MockedClient = elasticsearch_1.Client;
+const writeLog = (transport, doc) => transport.write(`${JSON.stringify(doc)}\n`);
+const waitForEvent = (emitter, eventName) => new Promise((resolve) => {
+    emitter.once(eventName, (value) => resolve(value));
+});
+describe('elastic transport resilience', () => {
+    beforeEach(() => {
+        jest.useRealTimers();
+        MockedClient.mockReset();
+    });
+    test('reports bulk failures without emitting terminal stream errors', async () => {
+        const bulk = jest
+            .fn()
+            .mockRejectedValueOnce(new Error('ES unavailable'))
+            .mockResolvedValueOnce({ errors: false, items: [] });
+        MockedClient.mockImplementation(() => ({ bulk }));
+        const transport = (0, elastic_transport_1.createElasticTransport)({
+            index: 'logs',
+            flushBytes: 1,
+            flushInterval: 10000,
+            requestTimeout: 10,
+        });
+        const streamError = jest.fn();
+        transport.on('error', streamError);
+        const failed = waitForEvent(transport, 'bulkError');
+        writeLog(transport, { message: 'first' });
+        await failed;
+        const inserted = waitForEvent(transport, 'insert');
+        writeLog(transport, { message: 'second' });
+        await expect(inserted).resolves.toEqual({ successful: 1, failed: 0 });
+        expect(streamError).not.toHaveBeenCalled();
+        expect(bulk).toHaveBeenCalledTimes(2);
+    });
+    test('watchdog releases a stuck bulk request so later logs can flush', async () => {
+        jest.useFakeTimers();
+        const bulk = jest
+            .fn()
+            .mockReturnValueOnce(new Promise(() => undefined))
+            .mockResolvedValueOnce({ errors: false, items: [] });
+        MockedClient.mockImplementation(() => ({ bulk }));
+        const transport = (0, elastic_transport_1.createElasticTransport)({
+            index: 'logs',
+            flushBytes: 1,
+            flushInterval: 10000,
+            requestTimeout: 1,
+        });
+        const timedOut = waitForEvent(transport, 'bulkError');
+        writeLog(transport, { message: 'stuck' });
+        await jest.advanceTimersByTimeAsync(5001);
+        await expect(timedOut).resolves.toMatchObject({
+            message: 'Elasticsearch bulk request timed out after 5001ms',
+        });
+        const inserted = waitForEvent(transport, 'insert');
+        writeLog(transport, { message: 'recovered' });
+        await expect(inserted).resolves.toEqual({ successful: 1, failed: 0 });
+        expect(bulk).toHaveBeenCalledTimes(2);
+    });
+});

package/dist/lib/elastic-transport.d.ts

@@ -24,4 +24,8 @@ export interface ElasticTransportOptions extends Pick<ClientOptions, 'node' | 'a
     rejectUnauthorized?: boolean;
     tls?: ClientOptions['tls'];
 }
-export declare const createElasticTransport: (opts?: ElasticTransportOptions) => NodeJS.ReadWriteStream;
+export interface ElasticTransportStream extends NodeJS.ReadWriteStream {
+    flush: () => Promise<void>;
+    close: () => Promise<void>;
+}
+export declare const createElasticTransport: (opts?: ElasticTransportOptions) => ElasticTransportStream;

package/dist/lib/elastic-transport.js

@@ -17,6 +17,7 @@ exports.createElasticTransport = void 0;
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 const split = require('split2');
 const elasticsearch_1 = require("@elastic/elasticsearch");
+const createTimeoutError = (timeoutMs) => new Error(`Elasticsearch bulk request timed out after ${timeoutMs}ms`);
 function setDateTimeString(value) {
     if (value !== null && typeof value === 'object' && 'time' in value) {
         const t = value.time;
@@ -33,42 +34,174 @@ function getIndexName(index, time) {
     }
     return index.replace('%{DATE}', time.substring(0, 10));
 }
-function initializeBulkHandler(opts, client, splitter, onFatalError) {
-    var _a, _b, _c, _d, _e, _f, _g;
+function createBulkSender(opts, client, splitter) {
+    var _a, _b, _c, _d, _e, _f, _g, _h;
     const esVersion = Number((_b = (_a = opts.esVersion) !== null && _a !== void 0 ? _a : opts['es-version']) !== null && _b !== void 0 ? _b : 7);
     const index = (_c = opts.index) !== null && _c !== void 0 ? _c : 'pino';
     const buildIndexName = typeof index === 'function' ? index : null;
     const opType = esVersion >= 7 ? undefined : undefined;
+    const flushBytes = (_e = (_d = opts.flushBytes) !== null && _d !== void 0 ? _d : opts['flush-bytes']) !== null && _e !== void 0 ? _e : 1000;
+    const flushInterval = (_g = (_f = opts.flushInterval) !== null && _f !== void 0 ? _f : opts['flush-interval']) !== null && _g !== void 0 ? _g : 3000;
+    const requestTimeout = (_h = opts.requestTimeout) !== null && _h !== void 0 ? _h : 30000;
+    const bulkWatchdogTimeout = requestTimeout + 5000;
+    let buffer = [];
+    let bufferedBytes = 0;
+    let timer;
+    let isFlushing = false;
+    let flushAgain = false;
     const indexName = (time = new Date().toISOString()) => buildIndexName ? buildIndexName(time) : getIndexName(index, time);
-    const bulkInsert = client.helpers.bulk({
-        datasource: splitter,
-        flushBytes: (_e = (_d = opts.flushBytes) !== null && _d !== void 0 ? _d : opts['flush-bytes']) !== null && _e !== void 0 ? _e : 1000,
-        flushInterval: (_g = (_f = opts.flushInterval) !== null && _f !== void 0 ? _f : opts['flush-interval']) !== null && _g !== void 0 ? _g : 3000,
-        refreshOnCompletion: false,
-        onDocument(doc) {
-            var _a, _b;
+    const clearFlushTimer = () => {
+        if (timer) {
+            clearTimeout(timer);
+            timer = undefined;
+        }
+    };
+    const scheduleFlush = () => {
+        var _a;
+        if (timer || buffer.length === 0) {
+            return;
+        }
+        timer = setTimeout(() => {
+            timer = undefined;
+            void flush();
+        }, flushInterval);
+        (_a = timer.unref) === null || _a === void 0 ? void 0 : _a.call(timer);
+    };
+    const buildOperation = (doc) => {
+        var _a, _b;
+        try {
             const d = doc;
             const date = (_b = (_a = d.time) !== null && _a !== void 0 ? _a : d['@timestamp']) !== null && _b !== void 0 ? _b : new Date().toISOString();
             if (opType === 'create') {
                 d['@timestamp'] = date;
             }
-            return {
-                index: {
-                    _index: indexName(date),
-                    op_type: opType,
+            return [
+                {
+                    index: {
+                        _index: indexName(date),
+                        op_type: opType,
+                    },
                 },
-            };
+                doc,
+            ];
+        }
+        catch (_c) {
+            return [
+                {
+                    index: {
+                        _index: indexName(),
+                        op_type: opType,
+                    },
+                },
+                doc,
+            ];
+        }
+    };
+    const emitDroppedDocument = (doc, cause) => {
+        const error = new Error('Dropped document');
+        error.document = doc;
+        error.cause = cause;
+        splitter.emit('insertError', error);
+    };
+    const emitBulkError = (err) => {
+        // Do not emit the standard stream "error" event for retryable bulk
+        // failures. Some stream consumers treat it as terminal, which can leave
+        // Pino writing into a poisoned stream while the process continues running.
+        splitter.emit('bulkError', err);
+    };
+    const bulkWithWatchdog = async (operations) => {
+        let timeout;
+        const bulkPromise = client.bulk({
+            operations,
+            refresh: false,
+            timeout: `${requestTimeout}ms`,
+        });
+        try {
+            return await Promise.race([
+                bulkPromise,
+                new Promise((_, reject) => {
+                    var _a;
+                    timeout = setTimeout(() => {
+                        reject(createTimeoutError(bulkWatchdogTimeout));
+                    }, bulkWatchdogTimeout);
+                    (_a = timeout.unref) === null || _a === void 0 ? void 0 : _a.call(timeout);
+                }),
+            ]);
+        }
+        finally {
+            if (timeout) {
+                clearTimeout(timeout);
+            }
+            // If the watchdog wins, the original request may reject later. Consume
+            // that rejection so a stale request cannot crash the app.
+            bulkPromise.catch(() => undefined);
+        }
+    };
+    const flush = async () => {
+        var _a, _b;
+        if (isFlushing) {
+            flushAgain = true;
+            return;
+        }
+        clearFlushTimer();
+        if (buffer.length === 0) {
+            return;
+        }
+        isFlushing = true;
+        const batch = buffer;
+        buffer = [];
+        bufferedBytes = 0;
+        try {
+            const operations = batch.flatMap(buildOperation);
+            const body = await bulkWithWatchdog(operations);
+            if (body.errors && Array.isArray(body.items)) {
+                body.items.forEach((item, index) => {
+                    const result = Object.values(item)[0];
+                    if (result === null || result === void 0 ? void 0 : result.error) {
+                        emitDroppedDocument(batch[index], result.error);
+                    }
+                });
+            }
+            splitter.emit('insert', {
+                successful: batch.length,
+                failed: body.errors
+                    ? ((_b = (_a = body.items) === null || _a === void 0 ? void 0 : _a.filter((item) => { var _a; return (_a = Object.values(item)[0]) === null || _a === void 0 ? void 0 : _a.error; }).length) !== null && _b !== void 0 ? _b : 0)
+                    : 0,
+            });
+        }
+        catch (err) {
+            emitBulkError(err);
+            // Drop the failed batch instead of wedging the stream. The next log line
+            // creates a fresh bulk request and can recover without a process restart.
+            batch.forEach((doc) => emitDroppedDocument(doc, err));
+        }
+        finally {
+            isFlushing = false;
+            if (flushAgain || buffer.length > 0) {
+                flushAgain = false;
+                scheduleFlush();
+                if (bufferedBytes >= flushBytes) {
+                    void flush();
+                }
+            }
+        }
+    };
+    return {
+        add(doc) {
+            buffer.push(doc);
+            bufferedBytes += Buffer.byteLength(JSON.stringify(doc));
+            if (bufferedBytes >= flushBytes) {
+                void flush();
+                return;
+            }
+            scheduleFlush();
         },
-        onDrop(doc) {
-            const error = new Error('Dropped document');
-            error.document = doc;
-            splitter.emit('insertError', error);
+        flush,
+        async close() {
+            clearFlushTimer();
+            await flush();
         },
-    });
-    bulkInsert.then((stats) => splitter.emit('insert', stats), (err) => {
-        splitter.emit('error', err);
-        onFatalError(err);
-    });
+    };
 }
 const createElasticTransport = (opts = {}) => {
     const splitter = split(function (line) {
@@ -119,56 +252,21 @@ const createElasticTransport = (opts = {}) => {
         clientOpts.ConnectionPool = opts.ConnectionPool;
     }
     const client = new elasticsearch_1.Client(clientOpts);
-    // CRITICAL FIX (pino-elasticsearch issues #140/#72): after retries are
-    // exhausted the bulk helper can stop consuming the stream while the process
-    // stays alive. Keep exactly one helper active and replace it after fatal
-    // helper failures instead of waiting for a server restart.
-    let isBulkHandlerActive = false;
-    let isRestartScheduled = false;
-    let isTransportClosed = false;
-    const pool = client.connectionPool;
+    const bulkSender = createBulkSender(opts, client, splitter);
+    splitter.on('data', (doc) => {
+        bulkSender.add(doc);
+    });
+    splitter.on('finish', () => {
+        void bulkSender.close();
+    });
     const splitterWithDestroy = splitter;
+    splitterWithDestroy.flush = bulkSender.flush;
+    splitterWithDestroy.close = bulkSender.close;
     const originalDestroy = splitterWithDestroy.destroy.bind(splitterWithDestroy);
-    const startBulkHandler = () => {
-        if (isTransportClosed || isBulkHandlerActive) {
-            return;
-        }
-        isBulkHandlerActive = true;
-        initializeBulkHandler(opts, client, splitter, () => {
-            isBulkHandlerActive = false;
-            scheduleBulkHandlerRestart();
-        });
-    };
-    const scheduleBulkHandlerRestart = () => {
-        var _a, _b, _c;
-        if (isTransportClosed || isRestartScheduled) {
-            return;
-        }
-        isRestartScheduled = true;
-        if (typeof pool.resurrect === 'function') {
-            pool.resurrect({ name: 'elasticsearch-js' });
-        }
-        const retryDelayMs = Math.min(Number((_b = (_a = opts.flushInterval) !== null && _a !== void 0 ? _a : opts['flush-interval']) !== null && _b !== void 0 ? _b : 3000), 5000);
-        const timer = setTimeout(() => {
-            isRestartScheduled = false;
-            startBulkHandler();
-        }, retryDelayMs);
-        (_c = timer.unref) === null || _c === void 0 ? void 0 : _c.call(timer);
-    };
     splitterWithDestroy.destroy = function (err) {
-        if (err && !isTransportClosed) {
-            scheduleBulkHandlerRestart();
-            return;
-        }
-        isTransportClosed = true;
+        void bulkSender.close();
         originalDestroy(err);
     };
-    client.diagnostic.on('resurrect', () => {
-        if (!isBulkHandlerActive) {
-            scheduleBulkHandlerRestart();
-        }
-    });
-    startBulkHandler();
-    return splitter;
+    return splitterWithDestroy;
 };
 exports.createElasticTransport = createElasticTransport;

package/dist/lib/logger.js

@@ -219,8 +219,12 @@ function registerShutdownHandlers() {
                         clearTimeout(timeout);
                         resolve();
                     });
-                    // Now trigger the flush
-                    esTransport === null || esTransport === void 0 ? void 0 : esTransport.end();
+                    esTransport === null || esTransport === void 0 ? void 0 : esTransport.flush().catch((error) => {
+                        console.error(`[Logger] Error flushing logs on ${signal}:`, error);
+                    }).finally(() => {
+                        // Now trigger stream shutdown
+                        esTransport === null || esTransport === void 0 ? void 0 : esTransport.end();
+                    });
                 });
             }
             catch (error) {
@@ -295,7 +299,7 @@ function getLogger(elasticConfig) {
             // Create transport with connection lifecycle fix (pino-elasticsearch #140)
             esTransport = (0, elastic_transport_1.createElasticTransport)(esConfig);
             // Handle Elasticsearch connection errors
-            esTransport.on('error', (err) => {
+            esTransport.on('bulkError', (err) => {
                 console.error('[Logger] Elasticsearch transport error:', err.message);
                 console.error('[Logger] Logs may not be reaching Kibana. Check Elasticsearch connection.');
             });

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@mhmdhammoud/meritt-utils",
-	"version": "1.6.0",
+	"version": "1.6.2",
 	"description": "",
 	"main": "./dist/index.js",
 	"private": false,

package/src/__tests__/elastic-transport.test.ts

@@ -0,0 +1,94 @@
+import { createElasticTransport } from '../lib/elastic-transport'
+import { Client } from '@elastic/elasticsearch'
+import { EventEmitter } from 'events'
+
+jest.mock('@elastic/elasticsearch', () => ({
+	Client: jest.fn(),
+}))
+
+const MockedClient = Client as jest.Mock
+
+const writeLog = (
+	transport: NodeJS.WritableStream,
+	doc: Record<string, unknown>
+) => transport.write(`${JSON.stringify(doc)}\n`)
+
+const waitForEvent = <T = unknown>(
+	emitter: EventEmitter,
+	eventName: string
+): Promise<T> =>
+	new Promise((resolve) => {
+		emitter.once(eventName, (value) => resolve(value as T))
+	})
+
+describe('elastic transport resilience', () => {
+	beforeEach(() => {
+		jest.useRealTimers()
+		MockedClient.mockReset()
+	})
+
+	test('reports bulk failures without emitting terminal stream errors', async () => {
+		const bulk = jest
+			.fn()
+			.mockRejectedValueOnce(new Error('ES unavailable'))
+			.mockResolvedValueOnce({ errors: false, items: [] })
+		MockedClient.mockImplementation(() => ({ bulk }))
+
+		const transport = createElasticTransport({
+			index: 'logs',
+			flushBytes: 1,
+			flushInterval: 10000,
+			requestTimeout: 10,
+		})
+		const streamError = jest.fn()
+		transport.on('error', streamError)
+
+		const failed = waitForEvent<Error>(transport, 'bulkError')
+		writeLog(transport, { message: 'first' })
+		await failed
+
+		const inserted = waitForEvent<{ successful: number; failed: number }>(
+			transport,
+			'insert'
+		)
+		writeLog(transport, { message: 'second' })
+
+		await expect(inserted).resolves.toEqual({ successful: 1, failed: 0 })
+		expect(streamError).not.toHaveBeenCalled()
+		expect(bulk).toHaveBeenCalledTimes(2)
+	})
+
+	test('watchdog releases a stuck bulk request so later logs can flush', async () => {
+		jest.useFakeTimers()
+
+		const bulk = jest
+			.fn()
+			.mockReturnValueOnce(new Promise(() => undefined))
+			.mockResolvedValueOnce({ errors: false, items: [] })
+		MockedClient.mockImplementation(() => ({ bulk }))
+
+		const transport = createElasticTransport({
+			index: 'logs',
+			flushBytes: 1,
+			flushInterval: 10000,
+			requestTimeout: 1,
+		})
+
+		const timedOut = waitForEvent<Error>(transport, 'bulkError')
+		writeLog(transport, { message: 'stuck' })
+
+		await jest.advanceTimersByTimeAsync(5001)
+		await expect(timedOut).resolves.toMatchObject({
+			message: 'Elasticsearch bulk request timed out after 5001ms',
+		})
+
+		const inserted = waitForEvent<{ successful: number; failed: number }>(
+			transport,
+			'insert'
+		)
+		writeLog(transport, { message: 'recovered' })
+
+		await expect(inserted).resolves.toEqual({ successful: 1, failed: 0 })
+		expect(bulk).toHaveBeenCalledTimes(2)
+	})
+})

package/src/lib/elastic-transport.ts

@@ -17,7 +17,6 @@ const split = require('split2') as (
 	fn: (line: string) => unknown,
 	opts?: { autoDestroy?: boolean }
 ) => NodeJS.ReadWriteStream
-import { Readable } from 'stream'
 import { Client } from '@elastic/elasticsearch'
 import type { ClientOptions } from '@elastic/elasticsearch'
 
@@ -50,6 +49,14 @@ interface LogDocument {
 	[k: string]: unknown
 }
 
+export interface ElasticTransportStream extends NodeJS.ReadWriteStream {
+	flush: () => Promise<void>
+	close: () => Promise<void>
+}
+
+const createTimeoutError = (timeoutMs: number): Error =>
+	new Error(`Elasticsearch bulk request timed out after ${timeoutMs}ms`)
+
 function setDateTimeString(value: unknown): string {
 	if (value !== null && typeof value === 'object' && 'time' in value) {
 		const t = (value as { time: unknown }).time
@@ -73,59 +80,205 @@ function getIndexName(
 	return index.replace('%{DATE}', time.substring(0, 10))
 }
 
-function initializeBulkHandler(
+function createBulkSender(
 	opts: ElasticTransportOptions,
 	client: Client,
-	splitter: NodeJS.ReadWriteStream,
-	onFatalError: (err: Error) => void
-): void {
+	splitter: NodeJS.ReadWriteStream
+): {
+	add: (doc: unknown) => void
+	flush: () => Promise<void>
+	close: () => Promise<void>
+} {
 	const esVersion = Number(opts.esVersion ?? opts['es-version'] ?? 7)
 	const index = opts.index ?? 'pino'
 	const buildIndexName = typeof index === 'function' ? index : null
 	const opType = esVersion >= 7 ? undefined : undefined
+	const flushBytes = opts.flushBytes ?? opts['flush-bytes'] ?? 1000
+	const flushInterval = opts.flushInterval ?? opts['flush-interval'] ?? 3000
+	const requestTimeout = opts.requestTimeout ?? 30000
+	const bulkWatchdogTimeout = requestTimeout + 5000
+
+	let buffer: unknown[] = []
+	let bufferedBytes = 0
+	let timer: NodeJS.Timeout | undefined
+	let isFlushing = false
+	let flushAgain = false
 
 	const indexName = (time = new Date().toISOString()) =>
 		buildIndexName ? buildIndexName(time) : getIndexName(index as string, time)
 
-	const bulkInsert = client.helpers.bulk({
-		datasource: splitter as unknown as Readable,
-		flushBytes: opts.flushBytes ?? opts['flush-bytes'] ?? 1000,
-		flushInterval: opts.flushInterval ?? opts['flush-interval'] ?? 3000,
-		refreshOnCompletion: false,
-		onDocument(doc: unknown) {
+	const clearFlushTimer = () => {
+		if (timer) {
+			clearTimeout(timer)
+			timer = undefined
+		}
+	}
+
+	const scheduleFlush = () => {
+		if (timer || buffer.length === 0) {
+			return
+		}
+		timer = setTimeout(() => {
+			timer = undefined
+			void flush()
+		}, flushInterval)
+		timer.unref?.()
+	}
+
+	const buildOperation = (doc: unknown): [Record<string, unknown>, unknown] => {
+		try {
 			const d = doc as LogDocument
 			const date = d.time ?? d['@timestamp'] ?? new Date().toISOString()
 			if (opType === 'create') {
 				d['@timestamp'] = date
 			}
-			return {
-				index: {
-					_index: indexName(date),
-					op_type: opType,
+			return [
+				{
+					index: {
+						_index: indexName(date),
+						op_type: opType,
+					},
+				},
+				doc,
+			]
+		} catch {
+			return [
+				{
+					index: {
+						_index: indexName(),
+						op_type: opType,
+					},
 				},
+				doc,
+			]
+		}
+	}
+
+	const emitDroppedDocument = (doc: unknown, cause?: unknown) => {
+		const error = new Error('Dropped document') as Error & {
+			document: unknown
+			cause?: unknown
+		}
+		error.document = doc
+		error.cause = cause
+		splitter.emit('insertError', error)
+	}
+
+	const emitBulkError = (err: unknown) => {
+		// Do not emit the standard stream "error" event for retryable bulk
+		// failures. Some stream consumers treat it as terminal, which can leave
+		// Pino writing into a poisoned stream while the process continues running.
+		splitter.emit('bulkError', err)
+	}
+
+	const bulkWithWatchdog = async (
+		operations: unknown[]
+	): Promise<{
+		errors?: boolean
+		items?: Array<Record<string, { error?: unknown }>>
+	}> => {
+		let timeout: NodeJS.Timeout | undefined
+		const bulkPromise = client.bulk({
+			operations,
+			refresh: false,
+			timeout: `${requestTimeout}ms`,
+		}) as Promise<{
+			errors?: boolean
+			items?: Array<Record<string, { error?: unknown }>>
+		}>
+
+		try {
+			return await Promise.race([
+				bulkPromise,
+				new Promise<never>((_, reject) => {
+					timeout = setTimeout(() => {
+						reject(createTimeoutError(bulkWatchdogTimeout))
+					}, bulkWatchdogTimeout)
+					timeout.unref?.()
+				}),
+			])
+		} finally {
+			if (timeout) {
+				clearTimeout(timeout)
 			}
-		},
-		onDrop(doc: unknown) {
-			const error = new Error('Dropped document') as Error & {
-				document: unknown
+			// If the watchdog wins, the original request may reject later. Consume
+			// that rejection so a stale request cannot crash the app.
+			bulkPromise.catch(() => undefined)
+		}
+	}
+
+	const flush = async (): Promise<void> => {
+		if (isFlushing) {
+			flushAgain = true
+			return
+		}
+		clearFlushTimer()
+		if (buffer.length === 0) {
+			return
+		}
+
+		isFlushing = true
+		const batch = buffer
+		buffer = []
+		bufferedBytes = 0
+
+		try {
+			const operations = batch.flatMap(buildOperation)
+			const body = await bulkWithWatchdog(operations)
+			if (body.errors && Array.isArray(body.items)) {
+				body.items.forEach((item, index) => {
+					const result = Object.values(item)[0]
+					if (result?.error) {
+						emitDroppedDocument(batch[index], result.error)
+					}
+				})
 			}
-			error.document = doc
-			splitter.emit('insertError', error)
-		},
-	})
 
-	bulkInsert.then(
-		(stats) => splitter.emit('insert', stats),
-		(err) => {
-			splitter.emit('error', err)
-			onFatalError(err)
+			splitter.emit('insert', {
+				successful: batch.length,
+				failed: body.errors
+					? (body.items?.filter((item) => Object.values(item)[0]?.error)
+							.length ?? 0)
+					: 0,
+			})
+		} catch (err) {
+			emitBulkError(err)
+			// Drop the failed batch instead of wedging the stream. The next log line
+			// creates a fresh bulk request and can recover without a process restart.
+			batch.forEach((doc) => emitDroppedDocument(doc, err))
+		} finally {
+			isFlushing = false
+			if (flushAgain || buffer.length > 0) {
+				flushAgain = false
+				scheduleFlush()
+				if (bufferedBytes >= flushBytes) {
+					void flush()
+				}
+			}
 		}
-	)
+	}
+
+	return {
+		add(doc: unknown) {
+			buffer.push(doc)
+			bufferedBytes += Buffer.byteLength(JSON.stringify(doc))
+			if (bufferedBytes >= flushBytes) {
+				void flush()
+				return
+			}
+			scheduleFlush()
+		},
+		flush,
+		async close() {
+			clearFlushTimer()
+			await flush()
+		},
+	}
 }
 
 export const createElasticTransport = (
 	opts: ElasticTransportOptions = {}
-): NodeJS.ReadWriteStream => {
+): ElasticTransportStream => {
 	const splitter = split(
 		function (this: NodeJS.ReadWriteStream, line: string) {
 			let value: unknown
@@ -179,71 +332,25 @@ export const createElasticTransport = (
 	}
 
 	const client = new Client(clientOpts)
+	const bulkSender = createBulkSender(opts, client, splitter)
 
-	// CRITICAL FIX (pino-elasticsearch issues #140/#72): after retries are
-	// exhausted the bulk helper can stop consuming the stream while the process
-	// stays alive. Keep exactly one helper active and replace it after fatal
-	// helper failures instead of waiting for a server restart.
-	let isBulkHandlerActive = false
-	let isRestartScheduled = false
-	let isTransportClosed = false
+	splitter.on('data', (doc) => {
+		bulkSender.add(doc)
+	})
+	splitter.on('finish', () => {
+		void bulkSender.close()
+	})
 
-	const pool = client.connectionPool as {
-		resurrect?: (opts: { name: string }) => void
-	}
-	const splitterWithDestroy = splitter as NodeJS.ReadWriteStream & {
+	const splitterWithDestroy = splitter as ElasticTransportStream & {
 		destroy: (err?: Error) => void
 	}
+	splitterWithDestroy.flush = bulkSender.flush
+	splitterWithDestroy.close = bulkSender.close
 	const originalDestroy = splitterWithDestroy.destroy.bind(splitterWithDestroy)
-
-	const startBulkHandler = () => {
-		if (isTransportClosed || isBulkHandlerActive) {
-			return
-		}
-		isBulkHandlerActive = true
-		initializeBulkHandler(opts, client, splitter, () => {
-			isBulkHandlerActive = false
-			scheduleBulkHandlerRestart()
-		})
-	}
-
-	const scheduleBulkHandlerRestart = () => {
-		if (isTransportClosed || isRestartScheduled) {
-			return
-		}
-		isRestartScheduled = true
-
-		if (typeof pool.resurrect === 'function') {
-			pool.resurrect({ name: 'elasticsearch-js' })
-		}
-
-		const retryDelayMs = Math.min(
-			Number(opts.flushInterval ?? opts['flush-interval'] ?? 3000),
-			5000
-		)
-		const timer = setTimeout(() => {
-			isRestartScheduled = false
-			startBulkHandler()
-		}, retryDelayMs)
-		timer.unref?.()
-	}
-
 	splitterWithDestroy.destroy = function (err?: Error) {
-		if (err && !isTransportClosed) {
-			scheduleBulkHandlerRestart()
-			return
-		}
-		isTransportClosed = true
+		void bulkSender.close()
 		originalDestroy(err)
 	}
 
-	client.diagnostic.on('resurrect', () => {
-		if (!isBulkHandlerActive) {
-			scheduleBulkHandlerRestart()
-		}
-	})
-
-	startBulkHandler()
-
-	return splitter
+	return splitterWithDestroy
 }

package/src/lib/logger.ts

@@ -1,7 +1,10 @@
 import { Logger as PinoLogger, pino, stdTimeFunctions } from 'pino'
 import * as dotenv from 'dotenv'
 import { hostname } from 'os'
-import { createElasticTransport } from './elastic-transport'
+import {
+	createElasticTransport,
+	ElasticTransportStream,
+} from './elastic-transport'
 import { getTraceContext } from './trace-store'
 import { LOG_LEVEL, LogEvent, ElasticConfig } from '../types'
 
@@ -140,7 +143,7 @@ let pinoLogger: PinoLogger
 /**
  * Elasticsearch transport instance - kept for cleanup
  */
-let esTransport: NodeJS.ReadWriteStream | null = null
+let esTransport: ElasticTransportStream | null = null
 
 /**
  * Flag to track if shutdown handlers are registered
@@ -233,8 +236,15 @@ function registerShutdownHandlers(): void {
 						resolve()
 					})
 
-					// Now trigger the flush
-					esTransport?.end()
+					esTransport
+						?.flush()
+						.catch((error) => {
+							console.error(`[Logger] Error flushing logs on ${signal}:`, error)
+						})
+						.finally(() => {
+							// Now trigger stream shutdown
+							esTransport?.end()
+						})
 				})
 			} catch (error) {
 				console.error(`[Logger] Error flushing logs on ${signal}:`, error)
@@ -333,7 +343,7 @@ function getLogger(elasticConfig?: ElasticConfig): PinoLogger {
 			esTransport = createElasticTransport(esConfig)
 
 			// Handle Elasticsearch connection errors
-			esTransport.on('error', (err: Error) => {
+			esTransport.on('bulkError', (err: Error) => {
 				console.error('[Logger] Elasticsearch transport error:', err.message)
 				console.error(
 					'[Logger] Logs may not be reaching Kibana. Check Elasticsearch connection.'