@mhmdhammoud/meritt-utils 1.5.5 → 1.5.7

package/dist/__tests__/logger.test.js

@@ -109,4 +109,28 @@ describe('route and format logs', () => {
             key1: 'val1',
         }));
     });
+    test('remap reserved elastic field names dynamically', () => {
+        //@ts-ignore
+        jest.spyOn(pino_1.pino, 'destination').mockReturnValue(PINO_DESTINATION);
+        //@ts-ignore
+        pino_1.pino.mockReturnValue(PINO);
+        const logger = new logger_1.default(LOGGER_NAME);
+        logger.info(LOG_EVENT, {
+            _id: 'abc123',
+            nested: {
+                _id: 'nested-1',
+                _index: 'bad-index',
+            },
+        });
+        expect(PINO.info).toHaveBeenCalledWith(expect.objectContaining({
+            mongo_id: 'abc123',
+            nested: expect.objectContaining({
+                mongo_id: 'nested-1',
+                es_index: 'bad-index',
+            }),
+        }));
+        expect(PINO.info).not.toHaveBeenCalledWith(expect.objectContaining({
+            _id: expect.anything(),
+        }));
+    });
 });

package/dist/lib/elastic-transport.js

@@ -56,7 +56,7 @@ function initializeBulkHandler(opts, client, splitter) {
         datasource: splitter,
         flushBytes: (_e = (_d = opts.flushBytes) !== null && _d !== void 0 ? _d : opts['flush-bytes']) !== null && _e !== void 0 ? _e : 1000,
         flushInterval: (_g = (_f = opts.flushInterval) !== null && _f !== void 0 ? _f : opts['flush-interval']) !== null && _g !== void 0 ? _g : 3000,
-        refreshOnCompletion: indexName(),
+        refreshOnCompletion: false,
         onDocument(doc) {
             var _a, _b;
             const d = doc;

package/dist/lib/logger.js

@@ -42,12 +42,107 @@ const trace_store_1 = require("./trace-store");
 dotenv.config();
 /** Convert camelCase to snake_case for Kibana/ECS-friendly field names */
 const toSnakeCase = (str) => str.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`);
+/**
+ * Elasticsearch reserves metadata fields (e.g. `_id`) and rejects documents
+ * containing them in payload body. Remap them to safe application fields.
+ */
+const RESERVED_ES_FIELD_ALIASES = {
+    _id: 'mongo_id',
+    _index: 'es_index',
+    _type: 'es_type',
+    _score: 'es_score',
+    _source: 'es_source',
+    _routing: 'es_routing',
+    _seq_no: 'es_seq_no',
+    _primary_term: 'es_primary_term',
+    _version: 'es_version',
+};
+/** Convert arbitrary field names into ES-safe flattened keys */
+const toSafeElasticFieldName = (key) => {
+    const normalized = toSnakeCase(key);
+    const mapped = RESERVED_ES_FIELD_ALIASES[normalized];
+    if (mapped) {
+        return mapped;
+    }
+    // Any leading underscore can conflict with ES internals, remap defensively.
+    return normalized.startsWith('_') ? `meta${normalized}` : normalized;
+};
+const isObjectIdLike = (v) => v !== null &&
+    typeof v === 'object' &&
+    'toHexString' in v &&
+    typeof v.toHexString === 'function';
 /** Check if value is a plain object (not Error, Date, Array, null) */
 const isPlainObject = (v) => v !== null &&
     typeof v === 'object' &&
     !Array.isArray(v) &&
     !(v instanceof Error) &&
     !(v instanceof Date);
+/**
+ * Recursively sanitize log values for Elasticsearch safety.
+ * - Remaps reserved key names (e.g. `_id` -> `mongo_id`)
+ * - Converts Error to a stable serializable shape
+ * - Converts ObjectId-like objects to hex strings
+ * - Prevents circular structure failures
+ */
+const sanitizeForElastic = (value, seen = new WeakSet()) => {
+    if (value === null ||
+        typeof value === 'string' ||
+        typeof value === 'number' ||
+        typeof value === 'boolean') {
+        return value;
+    }
+    if (value instanceof Date) {
+        return value.toISOString();
+    }
+    if (value instanceof Error) {
+        return { message: value.message, type: value.constructor.name };
+    }
+    if (isObjectIdLike(value)) {
+        return value.toHexString();
+    }
+    if (Array.isArray(value)) {
+        return value.map((item) => sanitizeForElastic(item, seen));
+    }
+    if (isPlainObject(value)) {
+        if (seen.has(value)) {
+            return '[Circular]';
+        }
+        seen.add(value);
+        const sanitizedObject = {};
+        for (const [k, v] of Object.entries(value)) {
+            sanitizedObject[toSafeElasticFieldName(k)] = sanitizeForElastic(v, seen);
+        }
+        return sanitizedObject;
+    }
+    // Fallback for class instances and non-plain objects.
+    return String(value);
+};
+/**
+ * Captures the call site from the stack when log event is missing.
+ * Returns file:line (e.g. product.service.ts:2266) for Kibana/Elasticsearch filtering.
+ */
+const getCallSiteForMissingLog = () => {
+    var _a;
+    try {
+        const stack = (_a = new Error().stack) !== null && _a !== void 0 ? _a : '';
+        const lines = stack.split('\n');
+        // First frame outside Logger / node_modules / meritt-utils
+        const appFrame = lines.find((line) => !line.includes('node_modules') &&
+            !line.includes('meritt-utils') &&
+            !line.includes('Logger.'));
+        if (!appFrame)
+            return undefined;
+        // Extract file:line e.g. "product.service.ts:2266"
+        const match = appFrame.match(/([^/\\]+\.(?:ts|js|tsx|jsx)):(\d+)/);
+        if (match) {
+            return `${match[1]}:${match[2]}`;
+        }
+        return appFrame.trim().slice(0, 100);
+    }
+    catch (_b) {
+        return undefined;
+    }
+};
 /**
  * Pino logger backend - singleton
  */
@@ -208,6 +303,13 @@ function getLogger(elasticConfig) {
             esTransport.on('insertError', (err) => {
                 console.error('[Logger] Elasticsearch insert error:', err.message);
                 console.error('[Logger] Some logs failed to index to Elasticsearch.');
+                if (err.document) {
+                    const docStr = JSON.stringify(err.document);
+                    const preview = docStr.length > 500
+                        ? `${docStr.substring(0, 500)}... (truncated)`
+                        : docStr;
+                    console.error('[Logger] Dropped document preview:', preview);
+                }
             });
             // Log successful connection (for debugging)
             esTransport.on('insert', () => {
@@ -259,35 +361,36 @@ class Logger {
     buildPayload(logLevel, logEvent, args) {
         var _a, _b;
         const isLocal = process.env.NODE_ENV === 'local' || process.env.NODE_ENV === 'test';
-        // ECS-aligned fields for Kibana Discover, Lens, alerts
+        // Defensive: missing Logs constant (undefined) crashes on logEvent.code
+        const useFallback = !logEvent || typeof logEvent !== 'object' || !('code' in logEvent);
+        const event = useFallback
+            ? { code: 'UNKNOWN', msg: 'Missing or invalid log event constant' }
+            : logEvent;
+        // ECS-aligned fields for Kibana (flat names to avoid mapping conflicts with existing indices)
         const ecs = {
-            'log.level': logLevel,
-            'log.logger': this._name,
-            'event.code': logEvent.code,
-            message: logEvent.msg,
-            service: {
-                name: (_a = process.env.SERVER_NICKNAME) !== null && _a !== void 0 ? _a : 'unknown',
-                environment: (_b = process.env.NODE_ENV) !== null && _b !== void 0 ? _b : 'development',
-            },
-            host: {
-                name: (0, os_1.hostname)(),
-            },
+            log_level: logLevel,
+            log_logger: this._name,
+            event_code: event.code,
+            message: event.msg,
+            service_name: (_a = process.env.SERVER_NICKNAME) !== null && _a !== void 0 ? _a : 'unknown',
+            service_environment: (_b = process.env.NODE_ENV) !== null && _b !== void 0 ? _b : 'development',
+            host_name: (0, os_1.hostname)(),
         };
         // Trace context for request-scoped correlation
         const trace = (0, trace_store_1.getTraceContext)();
         if (trace) {
-            ;
-            ecs.trace = { id: trace.traceId };
+            ecs.trace_id = trace.traceId;
         }
         // Structured context: flatten single plain object as top-level fields
+        // Sanitize values to avoid ES mapping conflicts (e.g. Error objects → serializable shape)
         let detail;
         if (args.length === 1 &&
             isPlainObject(args[0]) &&
             Object.keys(args[0]).length > 0) {
             const obj = args[0];
             for (const [k, v] of Object.entries(obj)) {
-                const key = toSnakeCase(k);
-                ecs[key] = v;
+                const key = toSafeElasticFieldName(k);
+                ecs[key] = sanitizeForElastic(v);
             }
         }
         else {
@@ -297,12 +400,19 @@ class Logger {
         const base = {
             ...ecs,
             component: this._name,
-            code: logEvent.code,
-            msg: logEvent.msg,
+            code: event.code,
+            msg: event.msg,
         };
         if (detail !== undefined) {
             base.detail = detail;
         }
+        // When fallback used: add call site for Kibana/Elasticsearch querying
+        if (useFallback) {
+            const callSite = getCallSiteForMissingLog();
+            if (callSite) {
+                base.missing_log_call_site = callSite;
+            }
+        }
         return base;
     }
     log(logLevel, logEvent, ...args) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@mhmdhammoud/meritt-utils",
-	"version": "1.5.5",
+	"version": "1.5.7",
 	"description": "",
 	"main": "./dist/index.js",
 	"private": false,

package/src/__tests__/logger.test.ts

@@ -89,4 +89,35 @@ describe('route and format logs', () => {
 			})
 		)
 	})
+
+	test('remap reserved elastic field names dynamically', () => {
+		//@ts-ignore
+		jest.spyOn(pino, 'destination').mockReturnValue(PINO_DESTINATION)
+		//@ts-ignore
+		pino.mockReturnValue(PINO)
+		const logger = new Logger(LOGGER_NAME)
+
+		logger.info(LOG_EVENT, {
+			_id: 'abc123',
+			nested: {
+				_id: 'nested-1',
+				_index: 'bad-index',
+			},
+		})
+
+		expect(PINO.info).toHaveBeenCalledWith(
+			expect.objectContaining({
+				mongo_id: 'abc123',
+				nested: expect.objectContaining({
+					mongo_id: 'nested-1',
+					es_index: 'bad-index',
+				}),
+			})
+		)
+		expect(PINO.info).not.toHaveBeenCalledWith(
+			expect.objectContaining({
+				_id: expect.anything(),
+			})
+		)
+	})
 })

package/src/lib/elastic-transport.ts

@@ -107,7 +107,7 @@ function initializeBulkHandler(
 		datasource: splitter as unknown as Readable,
 		flushBytes: opts.flushBytes ?? opts['flush-bytes'] ?? 1000,
 		flushInterval: opts.flushInterval ?? opts['flush-interval'] ?? 3000,
-		refreshOnCompletion: indexName(),
+		refreshOnCompletion: false,
 		onDocument(doc: unknown) {
 			const d = doc as LogDocument
 			const date = d.time ?? d['@timestamp'] ?? new Date().toISOString()

package/src/lib/logger.ts

@@ -1,9 +1,9 @@
-import {Logger as PinoLogger, pino, stdTimeFunctions} from 'pino'
+import { Logger as PinoLogger, pino, stdTimeFunctions } from 'pino'
 import * as dotenv from 'dotenv'
-import {hostname} from 'os'
-import {createElasticTransport} from './elastic-transport'
-import {getTraceContext} from './trace-store'
-import {LOG_LEVEL, LogEvent, ElasticConfig} from '../types'
+import { hostname } from 'os'
+import { createElasticTransport } from './elastic-transport'
+import { getTraceContext } from './trace-store'
+import { LOG_LEVEL, LogEvent, ElasticConfig } from '../types'
 
 dotenv.config()
 
@@ -11,6 +11,39 @@ dotenv.config()
 const toSnakeCase = (str: string): string =>
 	str.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)
 
+/**
+ * Elasticsearch reserves metadata fields (e.g. `_id`) and rejects documents
+ * containing them in payload body. Remap them to safe application fields.
+ */
+const RESERVED_ES_FIELD_ALIASES: Record<string, string> = {
+	_id: 'mongo_id',
+	_index: 'es_index',
+	_type: 'es_type',
+	_score: 'es_score',
+	_source: 'es_source',
+	_routing: 'es_routing',
+	_seq_no: 'es_seq_no',
+	_primary_term: 'es_primary_term',
+	_version: 'es_version',
+}
+
+/** Convert arbitrary field names into ES-safe flattened keys */
+const toSafeElasticFieldName = (key: string): string => {
+	const normalized = toSnakeCase(key)
+	const mapped = RESERVED_ES_FIELD_ALIASES[normalized]
+	if (mapped) {
+		return mapped
+	}
+	// Any leading underscore can conflict with ES internals, remap defensively.
+	return normalized.startsWith('_') ? `meta${normalized}` : normalized
+}
+
+const isObjectIdLike = (v: unknown): v is { toHexString: () => string } =>
+	v !== null &&
+	typeof v === 'object' &&
+	'toHexString' in v &&
+	typeof (v as { toHexString?: unknown }).toHexString === 'function'
+
 /** Check if value is a plain object (not Error, Date, Array, null) */
 const isPlainObject = (v: unknown): v is Record<string, unknown> =>
 	v !== null &&
@@ -19,6 +52,86 @@ const isPlainObject = (v: unknown): v is Record<string, unknown> =>
 	!(v instanceof Error) &&
 	!(v instanceof Date)
 
+/**
+ * Recursively sanitize log values for Elasticsearch safety.
+ * - Remaps reserved key names (e.g. `_id` -> `mongo_id`)
+ * - Converts Error to a stable serializable shape
+ * - Converts ObjectId-like objects to hex strings
+ * - Prevents circular structure failures
+ */
+const sanitizeForElastic = (
+	value: unknown,
+	seen: WeakSet<object> = new WeakSet<object>()
+): unknown => {
+	if (
+		value === null ||
+		typeof value === 'string' ||
+		typeof value === 'number' ||
+		typeof value === 'boolean'
+	) {
+		return value
+	}
+
+	if (value instanceof Date) {
+		return value.toISOString()
+	}
+
+	if (value instanceof Error) {
+		return { message: value.message, type: value.constructor.name }
+	}
+
+	if (isObjectIdLike(value)) {
+		return value.toHexString()
+	}
+
+	if (Array.isArray(value)) {
+		return value.map((item) => sanitizeForElastic(item, seen))
+	}
+
+	if (isPlainObject(value)) {
+		if (seen.has(value)) {
+			return '[Circular]'
+		}
+		seen.add(value)
+
+		const sanitizedObject: Record<string, unknown> = {}
+		for (const [k, v] of Object.entries(value)) {
+			sanitizedObject[toSafeElasticFieldName(k)] = sanitizeForElastic(v, seen)
+		}
+		return sanitizedObject
+	}
+
+	// Fallback for class instances and non-plain objects.
+	return String(value)
+}
+
+/**
+ * Captures the call site from the stack when log event is missing.
+ * Returns file:line (e.g. product.service.ts:2266) for Kibana/Elasticsearch filtering.
+ */
+const getCallSiteForMissingLog = (): string | undefined => {
+	try {
+		const stack = new Error().stack ?? ''
+		const lines = stack.split('\n')
+		// First frame outside Logger / node_modules / meritt-utils
+		const appFrame = lines.find(
+			(line) =>
+				!line.includes('node_modules') &&
+				!line.includes('meritt-utils') &&
+				!line.includes('Logger.')
+		)
+		if (!appFrame) return undefined
+		// Extract file:line e.g. "product.service.ts:2266"
+		const match = appFrame.match(/([^/\\]+\.(?:ts|js|tsx|jsx)):(\d+)/)
+		if (match) {
+			return `${match[1]}:${match[2]}`
+		}
+		return appFrame.trim().slice(0, 100)
+	} catch {
+		return undefined
+	}
+}
+
 /**
  * Pino logger backend - singleton
  */
@@ -228,9 +341,17 @@ function getLogger(elasticConfig?: ElasticConfig): PinoLogger {
 			})
 
 			// Handle insert errors (document indexing failures)
-			esTransport.on('insertError', (err: Error) => {
+			esTransport.on('insertError', (err: Error & { document?: unknown }) => {
 				console.error('[Logger] Elasticsearch insert error:', err.message)
 				console.error('[Logger] Some logs failed to index to Elasticsearch.')
+				if (err.document) {
+					const docStr = JSON.stringify(err.document)
+					const preview =
+						docStr.length > 500
+							? `${docStr.substring(0, 500)}... (truncated)`
+							: docStr
+					console.error('[Logger] Dropped document preview:', preview)
+				}
 			})
 
 			// Log successful connection (for debugging)
@@ -308,42 +429,50 @@ class Logger {
 	 * - Structured: Single plain object flattened as top-level snake_case fields (Kibana filterable)
 	 * - Trace: trace.id when running inside runWithTrace
 	 */
-	private buildPayload(logLevel: LOG_LEVEL, logEvent: LogEvent, args: unknown[]) {
+	private buildPayload(
+		logLevel: LOG_LEVEL,
+		logEvent: LogEvent,
+		args: unknown[]
+	) {
 		const isLocal =
 			process.env.NODE_ENV === 'local' || process.env.NODE_ENV === 'test'
 
-		// ECS-aligned fields for Kibana Discover, Lens, alerts
+		// Defensive: missing Logs constant (undefined) crashes on logEvent.code
+		const useFallback =
+			!logEvent || typeof logEvent !== 'object' || !('code' in logEvent)
+		const event = useFallback
+			? { code: 'UNKNOWN', msg: 'Missing or invalid log event constant' }
+			: logEvent
+
+		// ECS-aligned fields for Kibana (flat names to avoid mapping conflicts with existing indices)
 		const ecs: Record<string, unknown> = {
-			'log.level': logLevel,
-			'log.logger': this._name,
-			'event.code': logEvent.code,
-			message: logEvent.msg,
-			service: {
-				name: process.env.SERVER_NICKNAME ?? 'unknown',
-				environment: process.env.NODE_ENV ?? 'development',
-			},
-			host: {
-				name: hostname(),
-			},
+			log_level: logLevel,
+			log_logger: this._name,
+			event_code: event.code,
+			message: event.msg,
+			service_name: process.env.SERVER_NICKNAME ?? 'unknown',
+			service_environment: process.env.NODE_ENV ?? 'development',
+			host_name: hostname(),
 		}
 
 		// Trace context for request-scoped correlation
 		const trace = getTraceContext()
 		if (trace) {
-			;(ecs as Record<string, unknown>).trace = {id: trace.traceId}
+			ecs.trace_id = trace.traceId
 		}
 
 		// Structured context: flatten single plain object as top-level fields
+		// Sanitize values to avoid ES mapping conflicts (e.g. Error objects → serializable shape)
 		let detail: unknown
 		if (
 			args.length === 1 &&
 			isPlainObject(args[0]) &&
 			Object.keys(args[0]).length > 0
 		) {
-			const obj = args[0] as Record<string, unknown>
+			const obj = args[0]
 			for (const [k, v] of Object.entries(obj)) {
-				const key = toSnakeCase(k)
-				;(ecs as Record<string, unknown>)[key] = v
+				const key = toSafeElasticFieldName(k)
+				ecs[key] = sanitizeForElastic(v)
 			}
 		} else {
 			detail = isLocal ? args : JSON.stringify(args)
@@ -353,12 +482,19 @@ class Logger {
 		const base: Record<string, unknown> = {
 			...ecs,
 			component: this._name,
-			code: logEvent.code,
-			msg: logEvent.msg,
+			code: event.code,
+			msg: event.msg,
 		}
 		if (detail !== undefined) {
 			base.detail = detail
 		}
+		// When fallback used: add call site for Kibana/Elasticsearch querying
+		if (useFallback) {
+			const callSite = getCallSiteForMissingLog()
+			if (callSite) {
+				base.missing_log_call_site = callSite
+			}
+		}
 		return base
 	}
 
@@ -431,7 +567,7 @@ class Logger {
 			const result = await fn()
 			const durationMs = Date.now() - start
 			const payload = this.buildPayload('info', logEvent, [
-				{...context, duration_ms: durationMs, success: true},
+				{ ...context, duration_ms: durationMs, success: true },
 			])
 			this._logger.info(payload)
 			return result
@@ -439,10 +575,10 @@ class Logger {
 			const durationMs = Date.now() - start
 			const errObj =
 				error instanceof Error
-					? {error_message: error.message, error_type: error.constructor.name}
+					? { error_message: error.message, error_type: error.constructor.name }
 					: {}
 			const payload = this.buildPayload('error', logEvent, [
-				{...context, ...errObj, duration_ms: durationMs, success: false},
+				{ ...context, ...errObj, duration_ms: durationMs, success: false },
 			])
 			this._logger.error(payload)
 			throw error

package/src/lib/trace-store.ts

@@ -1,5 +1,5 @@
-import {AsyncLocalStorage} from 'async_hooks'
-import {randomUUID} from 'crypto'
+import { AsyncLocalStorage } from 'async_hooks'
+import { randomUUID } from 'crypto'
 
 /**
  * Trace context for request-scoped correlation in Kibana.
@@ -27,7 +27,7 @@ export const runWithTrace = async <T>(
 	traceId?: string
 ): Promise<T> => {
 	const id = traceId ?? randomUUID()
-	return traceStorage.run({traceId: id}, () => fn())
+	return traceStorage.run({ traceId: id }, () => fn())
 }
 
 /**
@@ -35,7 +35,7 @@ export const runWithTrace = async <T>(
  */
 export const runWithTraceSync = <T>(fn: () => T, traceId?: string): T => {
 	const id = traceId ?? randomUUID()
-	return traceStorage.run({traceId: id}, () => fn())
+	return traceStorage.run({ traceId: id }, () => fn())
 }
 
 /**