@mhmdhammoud/meritt-utils 1.6.0 → 1.6.1

package/dist/lib/elastic-transport.js

@@ -33,42 +33,141 @@ function getIndexName(index, time) {
     }
     return index.replace('%{DATE}', time.substring(0, 10));
 }
-function initializeBulkHandler(opts, client, splitter, onFatalError) {
+function createBulkSender(opts, client, splitter) {
     var _a, _b, _c, _d, _e, _f, _g;
     const esVersion = Number((_b = (_a = opts.esVersion) !== null && _a !== void 0 ? _a : opts['es-version']) !== null && _b !== void 0 ? _b : 7);
     const index = (_c = opts.index) !== null && _c !== void 0 ? _c : 'pino';
     const buildIndexName = typeof index === 'function' ? index : null;
     const opType = esVersion >= 7 ? undefined : undefined;
+    const flushBytes = (_e = (_d = opts.flushBytes) !== null && _d !== void 0 ? _d : opts['flush-bytes']) !== null && _e !== void 0 ? _e : 1000;
+    const flushInterval = (_g = (_f = opts.flushInterval) !== null && _f !== void 0 ? _f : opts['flush-interval']) !== null && _g !== void 0 ? _g : 3000;
+    let buffer = [];
+    let bufferedBytes = 0;
+    let timer;
+    let isFlushing = false;
+    let flushAgain = false;
     const indexName = (time = new Date().toISOString()) => buildIndexName ? buildIndexName(time) : getIndexName(index, time);
-    const bulkInsert = client.helpers.bulk({
-        datasource: splitter,
-        flushBytes: (_e = (_d = opts.flushBytes) !== null && _d !== void 0 ? _d : opts['flush-bytes']) !== null && _e !== void 0 ? _e : 1000,
-        flushInterval: (_g = (_f = opts.flushInterval) !== null && _f !== void 0 ? _f : opts['flush-interval']) !== null && _g !== void 0 ? _g : 3000,
-        refreshOnCompletion: false,
-        onDocument(doc) {
-            var _a, _b;
+    const clearFlushTimer = () => {
+        if (timer) {
+            clearTimeout(timer);
+            timer = undefined;
+        }
+    };
+    const scheduleFlush = () => {
+        var _a;
+        if (timer || buffer.length === 0) {
+            return;
+        }
+        timer = setTimeout(() => {
+            timer = undefined;
+            void flush();
+        }, flushInterval);
+        (_a = timer.unref) === null || _a === void 0 ? void 0 : _a.call(timer);
+    };
+    const buildOperation = (doc) => {
+        var _a, _b;
+        try {
             const d = doc;
             const date = (_b = (_a = d.time) !== null && _a !== void 0 ? _a : d['@timestamp']) !== null && _b !== void 0 ? _b : new Date().toISOString();
             if (opType === 'create') {
                 d['@timestamp'] = date;
             }
-            return {
-                index: {
-                    _index: indexName(date),
-                    op_type: opType,
+            return [
+                {
+                    index: {
+                        _index: indexName(date),
+                        op_type: opType,
+                    },
+                },
+                doc,
+            ];
+        }
+        catch (_c) {
+            return [
+                {
+                    index: {
+                        _index: indexName(),
+                        op_type: opType,
+                    },
                 },
-            };
+                doc,
+            ];
+        }
+    };
+    const emitDroppedDocument = (doc, cause) => {
+        const error = new Error('Dropped document');
+        error.document = doc;
+        error.cause = cause;
+        splitter.emit('insertError', error);
+    };
+    const flush = async () => {
+        var _a, _b;
+        if (isFlushing) {
+            flushAgain = true;
+            return;
+        }
+        clearFlushTimer();
+        if (buffer.length === 0) {
+            return;
+        }
+        isFlushing = true;
+        const batch = buffer;
+        buffer = [];
+        bufferedBytes = 0;
+        try {
+            const operations = batch.flatMap(buildOperation);
+            const response = await client.bulk({
+                operations,
+                refresh: false,
+                timeout: opts.requestTimeout ? `${opts.requestTimeout}ms` : undefined,
+            });
+            const body = response;
+            if (body.errors && Array.isArray(body.items)) {
+                body.items.forEach((item, index) => {
+                    const result = Object.values(item)[0];
+                    if (result === null || result === void 0 ? void 0 : result.error) {
+                        emitDroppedDocument(batch[index], result.error);
+                    }
+                });
+            }
+            splitter.emit('insert', {
+                successful: batch.length,
+                failed: body.errors ? ((_b = (_a = body.items) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0) : 0,
+            });
+        }
+        catch (err) {
+            splitter.emit('error', err);
+            // Drop the failed batch instead of wedging the stream. The next log line
+            // creates a fresh bulk request and can recover without a process restart.
+            batch.forEach((doc) => emitDroppedDocument(doc, err));
+        }
+        finally {
+            isFlushing = false;
+            if (flushAgain || buffer.length > 0) {
+                flushAgain = false;
+                scheduleFlush();
+                if (bufferedBytes >= flushBytes) {
+                    void flush();
+                }
+            }
+        }
+    };
+    return {
+        add(doc) {
+            buffer.push(doc);
+            bufferedBytes += Buffer.byteLength(JSON.stringify(doc));
+            if (bufferedBytes >= flushBytes) {
+                void flush();
+                return;
+            }
+            scheduleFlush();
         },
-        onDrop(doc) {
-            const error = new Error('Dropped document');
-            error.document = doc;
-            splitter.emit('insertError', error);
+        flush,
+        async close() {
+            clearFlushTimer();
+            await flush();
         },
-    });
-    bulkInsert.then((stats) => splitter.emit('insert', stats), (err) => {
-        splitter.emit('error', err);
-        onFatalError(err);
-    });
+    };
 }
 const createElasticTransport = (opts = {}) => {
     const splitter = split(function (line) {
@@ -119,56 +218,19 @@ const createElasticTransport = (opts = {}) => {
         clientOpts.ConnectionPool = opts.ConnectionPool;
     }
     const client = new elasticsearch_1.Client(clientOpts);
-    // CRITICAL FIX (pino-elasticsearch issues #140/#72): after retries are
-    // exhausted the bulk helper can stop consuming the stream while the process
-    // stays alive. Keep exactly one helper active and replace it after fatal
-    // helper failures instead of waiting for a server restart.
-    let isBulkHandlerActive = false;
-    let isRestartScheduled = false;
-    let isTransportClosed = false;
-    const pool = client.connectionPool;
+    const bulkSender = createBulkSender(opts, client, splitter);
+    splitter.on('data', (doc) => {
+        bulkSender.add(doc);
+    });
+    splitter.on('finish', () => {
+        void bulkSender.close();
+    });
     const splitterWithDestroy = splitter;
     const originalDestroy = splitterWithDestroy.destroy.bind(splitterWithDestroy);
-    const startBulkHandler = () => {
-        if (isTransportClosed || isBulkHandlerActive) {
-            return;
-        }
-        isBulkHandlerActive = true;
-        initializeBulkHandler(opts, client, splitter, () => {
-            isBulkHandlerActive = false;
-            scheduleBulkHandlerRestart();
-        });
-    };
-    const scheduleBulkHandlerRestart = () => {
-        var _a, _b, _c;
-        if (isTransportClosed || isRestartScheduled) {
-            return;
-        }
-        isRestartScheduled = true;
-        if (typeof pool.resurrect === 'function') {
-            pool.resurrect({ name: 'elasticsearch-js' });
-        }
-        const retryDelayMs = Math.min(Number((_b = (_a = opts.flushInterval) !== null && _a !== void 0 ? _a : opts['flush-interval']) !== null && _b !== void 0 ? _b : 3000), 5000);
-        const timer = setTimeout(() => {
-            isRestartScheduled = false;
-            startBulkHandler();
-        }, retryDelayMs);
-        (_c = timer.unref) === null || _c === void 0 ? void 0 : _c.call(timer);
-    };
     splitterWithDestroy.destroy = function (err) {
-        if (err && !isTransportClosed) {
-            scheduleBulkHandlerRestart();
-            return;
-        }
-        isTransportClosed = true;
+        void bulkSender.close();
         originalDestroy(err);
     };
-    client.diagnostic.on('resurrect', () => {
-        if (!isBulkHandlerActive) {
-            scheduleBulkHandlerRestart();
-        }
-    });
-    startBulkHandler();
     return splitter;
 };
 exports.createElasticTransport = createElasticTransport;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@mhmdhammoud/meritt-utils",
-	"version": "1.6.0",
+	"version": "1.6.1",
 	"description": "",
 	"main": "./dist/index.js",
 	"private": false,

package/src/lib/elastic-transport.ts

@@ -17,7 +17,6 @@ const split = require('split2') as (
 	fn: (line: string) => unknown,
 	opts?: { autoDestroy?: boolean }
 ) => NodeJS.ReadWriteStream
-import { Readable } from 'stream'
 import { Client } from '@elastic/elasticsearch'
 import type { ClientOptions } from '@elastic/elasticsearch'
 
@@ -73,54 +72,161 @@ function getIndexName(
 	return index.replace('%{DATE}', time.substring(0, 10))
 }
 
-function initializeBulkHandler(
+function createBulkSender(
 	opts: ElasticTransportOptions,
 	client: Client,
-	splitter: NodeJS.ReadWriteStream,
-	onFatalError: (err: Error) => void
-): void {
+	splitter: NodeJS.ReadWriteStream
+): {
+	add: (doc: unknown) => void
+	flush: () => Promise<void>
+	close: () => Promise<void>
+} {
 	const esVersion = Number(opts.esVersion ?? opts['es-version'] ?? 7)
 	const index = opts.index ?? 'pino'
 	const buildIndexName = typeof index === 'function' ? index : null
 	const opType = esVersion >= 7 ? undefined : undefined
+	const flushBytes = opts.flushBytes ?? opts['flush-bytes'] ?? 1000
+	const flushInterval = opts.flushInterval ?? opts['flush-interval'] ?? 3000
+
+	let buffer: unknown[] = []
+	let bufferedBytes = 0
+	let timer: NodeJS.Timeout | undefined
+	let isFlushing = false
+	let flushAgain = false
 
 	const indexName = (time = new Date().toISOString()) =>
 		buildIndexName ? buildIndexName(time) : getIndexName(index as string, time)
 
-	const bulkInsert = client.helpers.bulk({
-		datasource: splitter as unknown as Readable,
-		flushBytes: opts.flushBytes ?? opts['flush-bytes'] ?? 1000,
-		flushInterval: opts.flushInterval ?? opts['flush-interval'] ?? 3000,
-		refreshOnCompletion: false,
-		onDocument(doc: unknown) {
+	const clearFlushTimer = () => {
+		if (timer) {
+			clearTimeout(timer)
+			timer = undefined
+		}
+	}
+
+	const scheduleFlush = () => {
+		if (timer || buffer.length === 0) {
+			return
+		}
+		timer = setTimeout(() => {
+			timer = undefined
+			void flush()
+		}, flushInterval)
+		timer.unref?.()
+	}
+
+	const buildOperation = (doc: unknown): [Record<string, unknown>, unknown] => {
+		try {
 			const d = doc as LogDocument
 			const date = d.time ?? d['@timestamp'] ?? new Date().toISOString()
 			if (opType === 'create') {
 				d['@timestamp'] = date
 			}
-			return {
-				index: {
-					_index: indexName(date),
-					op_type: opType,
+			return [
+				{
+					index: {
+						_index: indexName(date),
+						op_type: opType,
+					},
+				},
+				doc,
+			]
+		} catch {
+			return [
+				{
+					index: {
+						_index: indexName(),
+						op_type: opType,
+					},
 				},
+				doc,
+			]
+		}
+	}
+
+	const emitDroppedDocument = (doc: unknown, cause?: unknown) => {
+		const error = new Error('Dropped document') as Error & {
+			document: unknown
+			cause?: unknown
+		}
+		error.document = doc
+		error.cause = cause
+		splitter.emit('insertError', error)
+	}
+
+	const flush = async (): Promise<void> => {
+		if (isFlushing) {
+			flushAgain = true
+			return
+		}
+		clearFlushTimer()
+		if (buffer.length === 0) {
+			return
+		}
+
+		isFlushing = true
+		const batch = buffer
+		buffer = []
+		bufferedBytes = 0
+
+		try {
+			const operations = batch.flatMap(buildOperation)
+			const response = await client.bulk({
+				operations,
+				refresh: false,
+				timeout: opts.requestTimeout ? `${opts.requestTimeout}ms` : undefined,
+			})
+
+			const body = response as {
+				errors?: boolean
+				items?: Array<Record<string, { error?: unknown }>>
 			}
-		},
-		onDrop(doc: unknown) {
-			const error = new Error('Dropped document') as Error & {
-				document: unknown
+			if (body.errors && Array.isArray(body.items)) {
+				body.items.forEach((item, index) => {
+					const result = Object.values(item)[0]
+					if (result?.error) {
+						emitDroppedDocument(batch[index], result.error)
+					}
+				})
 			}
-			error.document = doc
-			splitter.emit('insertError', error)
-		},
-	})
 
-	bulkInsert.then(
-		(stats) => splitter.emit('insert', stats),
-		(err) => {
+			splitter.emit('insert', {
+				successful: batch.length,
+				failed: body.errors ? (body.items?.length ?? 0) : 0,
+			})
+		} catch (err) {
 			splitter.emit('error', err)
-			onFatalError(err)
+			// Drop the failed batch instead of wedging the stream. The next log line
+			// creates a fresh bulk request and can recover without a process restart.
+			batch.forEach((doc) => emitDroppedDocument(doc, err))
+		} finally {
+			isFlushing = false
+			if (flushAgain || buffer.length > 0) {
+				flushAgain = false
+				scheduleFlush()
+				if (bufferedBytes >= flushBytes) {
+					void flush()
+				}
+			}
 		}
-	)
+	}
+
+	return {
+		add(doc: unknown) {
+			buffer.push(doc)
+			bufferedBytes += Buffer.byteLength(JSON.stringify(doc))
+			if (bufferedBytes >= flushBytes) {
+				void flush()
+				return
+			}
+			scheduleFlush()
+		},
+		flush,
+		async close() {
+			clearFlushTimer()
+			await flush()
+		},
+	}
 }
 
 export const createElasticTransport = (
@@ -179,71 +285,23 @@ export const createElasticTransport = (
 	}
 
 	const client = new Client(clientOpts)
+	const bulkSender = createBulkSender(opts, client, splitter)
 
-	// CRITICAL FIX (pino-elasticsearch issues #140/#72): after retries are
-	// exhausted the bulk helper can stop consuming the stream while the process
-	// stays alive. Keep exactly one helper active and replace it after fatal
-	// helper failures instead of waiting for a server restart.
-	let isBulkHandlerActive = false
-	let isRestartScheduled = false
-	let isTransportClosed = false
+	splitter.on('data', (doc) => {
+		bulkSender.add(doc)
+	})
+	splitter.on('finish', () => {
+		void bulkSender.close()
+	})
 
-	const pool = client.connectionPool as {
-		resurrect?: (opts: { name: string }) => void
-	}
 	const splitterWithDestroy = splitter as NodeJS.ReadWriteStream & {
 		destroy: (err?: Error) => void
 	}
 	const originalDestroy = splitterWithDestroy.destroy.bind(splitterWithDestroy)
-
-	const startBulkHandler = () => {
-		if (isTransportClosed || isBulkHandlerActive) {
-			return
-		}
-		isBulkHandlerActive = true
-		initializeBulkHandler(opts, client, splitter, () => {
-			isBulkHandlerActive = false
-			scheduleBulkHandlerRestart()
-		})
-	}
-
-	const scheduleBulkHandlerRestart = () => {
-		if (isTransportClosed || isRestartScheduled) {
-			return
-		}
-		isRestartScheduled = true
-
-		if (typeof pool.resurrect === 'function') {
-			pool.resurrect({ name: 'elasticsearch-js' })
-		}
-
-		const retryDelayMs = Math.min(
-			Number(opts.flushInterval ?? opts['flush-interval'] ?? 3000),
-			5000
-		)
-		const timer = setTimeout(() => {
-			isRestartScheduled = false
-			startBulkHandler()
-		}, retryDelayMs)
-		timer.unref?.()
-	}
-
 	splitterWithDestroy.destroy = function (err?: Error) {
-		if (err && !isTransportClosed) {
-			scheduleBulkHandlerRestart()
-			return
-		}
-		isTransportClosed = true
+		void bulkSender.close()
 		originalDestroy(err)
 	}
 
-	client.diagnostic.on('resurrect', () => {
-		if (!isBulkHandlerActive) {
-			scheduleBulkHandlerRestart()
-		}
-	})
-
-	startBulkHandler()
-
 	return splitter
 }