@aichatwar/shared 1.0.169 → 1.0.171

package/build/events/kafka/baseListener.d.ts

@@ -1,14 +1,31 @@
-import { Consumer, EachMessagePayload } from "kafkajs";
+import { Consumer, Producer, EachMessagePayload } from "kafkajs";
 import { BaseEvent } from '../baseEvent';
 export type { EachMessagePayload };
 export declare abstract class Listener<T extends BaseEvent> {
     abstract topic: T['subject'];
     abstract groupId: string;
     abstract onMessage(data: T['data'], payload: EachMessagePayload): Promise<void>;
+    private static registry;
+    /**
+     * Returns true only if every registered listener's consumer is connected.
+     * Wire this into your /ready endpoint alongside the MongoDB check.
+     */
+    static allHealthy(): boolean;
+    /**
+     * Returns per-listener health details for diagnostics.
+     */
+    static healthDetails(): Array<{
+        topic: string;
+        groupId: string;
+        healthy: boolean;
+    }>;
     protected consumer: Consumer;
+    protected dlqProducer?: Producer;
     protected ackDeadline: number;
     protected fromBeginning: boolean;
     protected maxEventAgeMs: number;
+    protected maxMessageRetries: number;
+    protected messageRetryBaseDelayMs: number;
     private currentPayload?;
     private retryCount;
     private readonly maxInitialRetries;
@@ -16,9 +33,19 @@ export declare abstract class Listener<T extends BaseEvent> {
     private isListening;
     private crashHandlerSetup;
     private crashRestartCount;
+    private readonly maxCrashRestarts;
     private readonly maxCrashRestartDelay;
-    constructor(consumer: Consumer);
+    private lastSuccessfulMessageAt;
+    /** Expose consumer health for readiness probes */
+    get healthy(): boolean;
+    constructor(consumer: Consumer, dlqProducer?: Producer);
     private setupCrashHandler;
+    /**
+     * Detects errors that are guaranteed to fail on every retry (poison messages).
+     * These should go straight to DLQ without wasting retry attempts.
+     */
+    private isPermanentError;
+    private publishToDlq;
     ack(payload?: EachMessagePayload): Promise<void>;
     listen(): Promise<void>;
 }

package/build/events/kafka/baseListener.js

@@ -11,18 +11,47 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Listener = void 0;
 class Listener {
-    constructor(consumer) {
+    /**
+     * Returns true only if every registered listener's consumer is connected.
+     * Wire this into your /ready endpoint alongside the MongoDB check.
+     */
+    static allHealthy() {
+        if (Listener.registry.length === 0)
+            return true;
+        return Listener.registry.every(l => l.healthy);
+    }
+    /**
+     * Returns per-listener health details for diagnostics.
+     */
+    static healthDetails() {
+        return Listener.registry.map(l => ({
+            topic: l.topic,
+            groupId: l.groupId,
+            healthy: l.healthy,
+        }));
+    }
+    /** Expose consumer health for readiness probes */
+    get healthy() {
+        return this.isListening;
+    }
+    constructor(consumer, dlqProducer) {
         this.ackDeadline = 5 * 1000; // 5 seconds
         this.fromBeginning = false; // Override in subclasses to read from beginning
         this.maxEventAgeMs = 0; // 0 = disabled. When set, messages older than this are auto-acked and skipped.
+        this.maxMessageRetries = 3; // Message-level retries before DLQ
+        this.messageRetryBaseDelayMs = 1000;
         this.retryCount = 0;
         this.maxInitialRetries = 3; // Show detailed retry logs for first 3 attempts
         this.maxRetryDelay = 60000; // Cap delay at 60 seconds
         this.isListening = false; // Track if listener is active
         this.crashHandlerSetup = false; // Track if crash handler has been set up
         this.crashRestartCount = 0;
+        this.maxCrashRestarts = 5; // Exit process after this many consecutive crashes
         this.maxCrashRestartDelay = 120000; // 2 minutes cap
+        this.lastSuccessfulMessageAt = 0; // Timestamp of last successfully processed message
         this.consumer = consumer;
+        this.dlqProducer = dlqProducer;
+        Listener.registry.push(this);
         this.setupCrashHandler();
     }
     setupCrashHandler() {
@@ -37,23 +66,91 @@ class Listener {
                 return;
             }
             this.crashRestartCount++;
+            this.isListening = false;
+            if (this.crashRestartCount >= this.maxCrashRestarts) {
+                console.error(`💀 [${this.topic}] Consumer crashed ${this.crashRestartCount} times without recovery. ` +
+                    `Exiting process to let Kubernetes restart the pod with a fresh state.`);
+                process.exit(1);
+            }
             const delay = Math.min(5000 * Math.pow(2, Math.min(this.crashRestartCount - 1, 5)), this.maxCrashRestartDelay);
-            console.error(`[${this.topic}] Consumer crashed (restart #${this.crashRestartCount}, ` +
+            console.error(`[${this.topic}] Consumer crashed (restart #${this.crashRestartCount}/${this.maxCrashRestarts}, ` +
                 `retrying in ${delay}ms):`, error);
-            this.isListening = false;
             try {
                 yield this.consumer.disconnect();
             }
             catch (_) { /* best effort */ }
             setTimeout(() => {
-                console.log(`[${this.topic}] Auto-restarting consumer after crash (attempt #${this.crashRestartCount})...`);
+                console.log(`[${this.topic}] Auto-restarting consumer after crash (attempt #${this.crashRestartCount}/${this.maxCrashRestarts})...`);
                 this.listen().catch((err) => {
                     console.error(`[${this.topic}] Auto-restart failed:`, err);
+                    console.error(`💀 [${this.topic}] Auto-restart listen() threw. Exiting process to let Kubernetes restart the pod.`);
+                    process.exit(1);
                 });
             }, delay);
         }));
+        this.consumer.on('consumer.disconnect', () => {
+            if (this.isListening) {
+                console.warn(`⚠️ [${this.topic}] Consumer disconnected unexpectedly`);
+                this.isListening = false;
+            }
+        });
         this.crashHandlerSetup = true;
     }
+    /**
+     * Detects errors that are guaranteed to fail on every retry (poison messages).
+     * These should go straight to DLQ without wasting retry attempts.
+     */
+    isPermanentError(error) {
+        if (!error)
+            return false;
+        // Mongoose VersionError — document version has advanced, stale event can never match
+        if (error.name === 'VersionError')
+            return true;
+        // MongoDB E11000 duplicate key — record already exists
+        if (error.code === 11000)
+            return true;
+        // NotFoundError thrown in listener — document doesn't exist (out-of-order event)
+        if (error.name === 'NotFoundError' || error.statusCode === 404)
+            return true;
+        // ValidationError — schema violation, data itself is invalid
+        if (error.name === 'ValidationError')
+            return true;
+        return false;
+    }
+    publishToDlq(data, error, partition, offset) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.dlqProducer)
+                return;
+            const dlqTopic = `${this.topic}.dlq`;
+            try {
+                yield this.dlqProducer.send({
+                    topic: dlqTopic,
+                    messages: [{
+                            key: (data === null || data === void 0 ? void 0 : data.id) || (data === null || data === void 0 ? void 0 : data.messageId) || null,
+                            value: JSON.stringify({
+                                originalData: data,
+                                error: (error === null || error === void 0 ? void 0 : error.message) || 'Unknown error',
+                                errorName: (error === null || error === void 0 ? void 0 : error.name) || 'Error',
+                                errorCode: error === null || error === void 0 ? void 0 : error.code,
+                                topic: this.topic,
+                                groupId: this.groupId,
+                                partition,
+                                offset,
+                                failedAt: new Date().toISOString(),
+                            }),
+                            timestamp: Date.now().toString(),
+                        }],
+                });
+                console.warn(`☠️ [${this.topic}] Poison message sent to DLQ ${dlqTopic} ` +
+                    `(partition: ${partition}, offset: ${offset}, error: ${(error === null || error === void 0 ? void 0 : error.name) || 'Error'}: ${error === null || error === void 0 ? void 0 : error.message})`);
+            }
+            catch (dlqErr) {
+                console.error(`[${this.topic}] CRITICAL: Failed to publish to DLQ ${dlqTopic}: ${dlqErr.message}. ` +
+                    `Original error: ${error === null || error === void 0 ? void 0 : error.message}. Message at partition ${partition} offset ${offset} will be retried by crash handler.`);
+                throw error;
+            }
+        });
+    }
     // Manual acknowledgment method
     // Accepts optional payload for custom consumer patterns (e.g., realtime-gateway)
     ack(payload) {
@@ -67,6 +164,12 @@ class Listener {
                     partition: targetPayload.partition,
                     offset: (BigInt(targetPayload.message.offset) + BigInt(1)).toString()
                 }]);
+            // Reset crash counter on successful processing — proves the consumer is healthy
+            if (this.crashRestartCount > 0) {
+                console.log(`[${this.topic}] Crash counter reset (was ${this.crashRestartCount}) after successful message processing`);
+                this.crashRestartCount = 0;
+            }
+            this.lastSuccessfulMessageAt = Date.now();
             console.log(`Message manually acknowledged for topic: ${this.topic}`);
         });
     }
@@ -95,14 +198,7 @@ class Listener {
                 }
                 // Reset retry count on successful connection
                 this.retryCount = 0;
-                console.log(`🚀 [${this.topic}] About to call consumer.run() with groupId: ${this.groupId}`);
-                console.log(`🚀 [${this.topic}] Starting consumer.run() with groupId: ${this.groupId}`);
-                console.log(`🚀 [${this.topic}] Consumer configuration:`, {
-                    groupId: this.groupId,
-                    topic: this.topic,
-                    fromBeginning: this.fromBeginning,
-                    autoCommit: false,
-                });
+                console.log(`🚀 [${this.topic}] Starting consumer.run() with groupId: ${this.groupId}, dlq: ${this.dlqProducer ? 'enabled' : 'disabled'}`);
                 yield this.consumer.run({
                     // CRITICAL: Disable auto-commit to prevent message loss during rebalancing
                     // Offsets are only committed when we explicitly call ack()
@@ -119,8 +215,6 @@ class Listener {
                                 }]);
                             return;
                         }
-                        // Commented out key logging as requested
-                        // console.log(`📨 [${this.topic}] Message received -> groupId: ${this.groupId}, partition: ${payload.partition}, offset: ${payload.message.offset}, key: ${payload.message.key?.toString() || 'none'}`);
                         console.log(`📨 [${this.topic}] Message received -> groupId: ${this.groupId}, partition: ${payload.partition}, offset: ${payload.message.offset}, value length: ${payload.message.value.toString().length}`);
                         // Skip stale messages when maxEventAgeMs is configured
                         if (this.maxEventAgeMs > 0 && payload.message.timestamp) {
@@ -136,17 +230,74 @@ class Listener {
                         }
                         // Store current payload for manual ack
                         this.currentPayload = payload;
+                        let data;
+                        try {
+                            data = JSON.parse(payload.message.value.toString());
+                        }
+                        catch (parseErr) {
+                            console.error(`❌ [${this.topic}] JSON parse error at offset ${payload.message.offset} — skipping malformed message`);
+                            if (this.dlqProducer) {
+                                yield this.publishToDlq(payload.message.value.toString(), parseErr, payload.partition, payload.message.offset);
+                            }
+                            yield this.consumer.commitOffsets([{
+                                    topic: payload.topic,
+                                    partition: payload.partition,
+                                    offset: (BigInt(payload.message.offset) + BigInt(1)).toString()
+                                }]);
+                            this.currentPayload = undefined;
+                            return;
+                        }
                         try {
-                            const data = JSON.parse(payload.message.value.toString());
                             yield this.onMessage(data, payload);
                             // Note: Child listeners MUST call this.ack() manually after successful processing
                             // If they don't call ack(), the message will be redelivered after session timeout
                             // This ensures at-least-once delivery semantics
                         }
                         catch (error) {
+                            // --- DLQ-based poison message handling ---
+                            // If a DLQ producer is available, we handle the error here instead of
+                            // crashing the consumer. The message is retried N times, then published
+                            // to a DLQ topic and acked so the consumer can move on.
+                            if (this.dlqProducer) {
+                                if (this.isPermanentError(error)) {
+                                    console.error(`❌ [${this.topic}] Permanent error (${error.name || 'Error'}) at partition ${payload.partition} ` +
+                                        `offset ${payload.message.offset} — skipping retries, sending to DLQ`);
+                                    yield this.publishToDlq(data, error, payload.partition, payload.message.offset);
+                                    yield this.ack(payload);
+                                    this.currentPayload = undefined;
+                                    return;
+                                }
+                                // Retry transient errors with backoff
+                                let lastRetryError = error;
+                                for (let attempt = 2; attempt <= this.maxMessageRetries; attempt++) {
+                                    const delay = this.messageRetryBaseDelayMs * Math.pow(2, attempt - 2);
+                                    console.warn(`[${this.topic}] Retry ${attempt}/${this.maxMessageRetries} for offset ${payload.message.offset} ` +
+                                        `in ${delay}ms (error: ${lastRetryError.message})`);
+                                    yield new Promise(r => setTimeout(r, delay));
+                                    try {
+                                        this.currentPayload = payload;
+                                        yield this.onMessage(data, payload);
+                                        this.currentPayload = undefined;
+                                        return; // Retry succeeded — onMessage called ack()
+                                    }
+                                    catch (retryErr) {
+                                        lastRetryError = retryErr;
+                                        if (this.isPermanentError(retryErr)) {
+                                            console.error(`❌ [${this.topic}] Permanent error on retry ${attempt} — sending to DLQ`);
+                                            break;
+                                        }
+                                    }
+                                }
+                                // All retries exhausted — DLQ + ack
+                                console.error(`❌ [${this.topic}] All ${this.maxMessageRetries} retries exhausted for offset ${payload.message.offset} — sending to DLQ`);
+                                yield this.publishToDlq(data, lastRetryError, payload.partition, payload.message.offset);
+                                yield this.ack(payload);
+                                this.currentPayload = undefined;
+                                return;
+                            }
+                            // --- Legacy behavior (no DLQ producer) ---
+                            // Re-throw to trigger the crash handler, which will eventually process.exit(1)
                             console.error(`❌ [${this.topic}] Error processing message for topic: ${this.topic}, offset: ${payload.message.offset}`, error);
-                            // In case of error, we don't commit the offset, so the message will be redelivered
-                            // This ensures failed messages are retried
                             throw error;
                         }
                         finally {
@@ -198,3 +349,4 @@ class Listener {
     }
 }
 exports.Listener = Listener;
+Listener.registry = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aichatwar/shared",
-  "version": "1.0.169",
+  "version": "1.0.171",
   "main": "./build/index.js",
   "typs": "./build/index.d.ts",
   "files": [