@lucaapp/service-utils 5.10.0 → 5.11.0

package/dist/lib/kafka/defaultPoisonHandler.d.ts

@@ -0,0 +1,13 @@
+import type { CustomTopicPoisonHandler, GenericKafkaEvent } from './types';
+type PoisonHandlerDeps = {
+    logger: {
+        error: (obj: Record<string, unknown>, msg: string) => void;
+    };
+    produceCustom: (topicName: string, key: string, value: GenericKafkaEvent) => Promise<void>;
+};
+/**
+ * Creates a poison handler that logs the error and produces the raw
+ * payload to a DLQ topic named `${topicName}-dlq`.
+ */
+declare const createDefaultPoisonHandler: (topicName: string, deps: PoisonHandlerDeps) => CustomTopicPoisonHandler;
+export { createDefaultPoisonHandler };

package/dist/lib/kafka/defaultPoisonHandler.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDefaultPoisonHandler = void 0;
+/**
+ * Creates a poison handler that logs the error and produces the raw
+ * payload to a DLQ topic named `${topicName}-dlq`.
+ */
+const createDefaultPoisonHandler = (topicName, deps) => {
+    const dlqTopicName = `${topicName}-dlq`;
+    return async (ctx) => {
+        const errorMessage = ctx.error instanceof Error ? ctx.error.message : String(ctx.error);
+        deps.logger.error({
+            topic: topicName,
+            dlqTopic: dlqTopicName,
+            partition: ctx.partition,
+            error: errorMessage,
+            key: ctx.message.key?.toString(),
+        }, 'Poison message detected, forwarding to DLQ');
+        try {
+            await deps.produceCustom(dlqTopicName, ctx.message.key?.toString() ?? 'unknown', {
+                id: 'dlq',
+                type: 'create',
+                entity: {
+                    rawPayload: ctx.rawPayloadUtf8,
+                    error: errorMessage,
+                    originalTopic: ctx.primaryTopicFullName,
+                    partition: ctx.partition,
+                    timestamp: new Date().toISOString(),
+                },
+            });
+        }
+        catch (dlqError) {
+            deps.logger.error({ error: dlqError, topic: dlqTopicName }, 'Failed to produce to DLQ topic');
+        }
+    };
+};
+exports.createDefaultPoisonHandler = createDefaultPoisonHandler;

package/dist/lib/kafka/index.d.ts

@@ -1,3 +1,3 @@
 export { KafkaClient } from './kafkaClient';
 export { KafkaTopic } from './events';
-export type { KafkaEvent, GenericKafkaEvent, EventPayloadHandler, GenericEventPayloadHandler, KafkaConfiguration, CustomTopicConfig, } from './types';
+export type { KafkaEvent, GenericKafkaEvent, EventPayloadHandler, GenericEventPayloadHandler, KafkaConfiguration, CustomTopicConfig, CustomTopicPoisonContext, CustomTopicPoisonHandler, } from './types';

package/dist/lib/kafka/kafkaClient.d.ts

@@ -1,7 +1,7 @@
 import { Consumer } from 'kafkajs';
 import { Logger } from 'pino';
 import { ServiceIdentity } from '../serviceIdentity';
-import type { EventPayloadHandler, GenericEventPayloadHandler, KafkaConfiguration, KafkaEvent, GenericKafkaEvent, CustomTopicConfig } from './types';
+import type { EventPayloadHandler, GenericEventPayloadHandler, KafkaConfiguration, KafkaEvent, GenericKafkaEvent, CustomTopicConfig, CustomTopicPoisonHandler } from './types';
 import { KafkaTopic } from './events';
 declare class KafkaClient {
     private readonly environment;
@@ -42,7 +42,8 @@ declare class KafkaClient {
     /**
      * Consume messages from custom topic
      */
-    consumeCustom: <T = unknown>(topicName: string, handler: GenericEventPayloadHandler<T>, fromBeginning?: boolean) => Promise<Consumer>;
+    consumeCustom: <T = unknown>(topicName: string, handler: GenericEventPayloadHandler<T>, fromBeginning?: boolean, onPoison?: boolean | CustomTopicPoisonHandler) => Promise<Consumer>;
+    private defaultPoisonHandler;
     private encryptCustomValue;
     private decryptCustomValue;
     private parseCustomValue;

package/dist/lib/kafka/kafkaClient.js

@@ -38,6 +38,7 @@ const kafkajs_1 = require("kafkajs");
 const jose = __importStar(require("jose"));
 const util_1 = require("util");
 const serviceIdentity_1 = require("../serviceIdentity");
+const defaultPoisonHandler_1 = require("./defaultPoisonHandler");
 const events_1 = require("./events");
 const utils_1 = require("../../utils/utils");
 const metrics_1 = require("../metrics");
@@ -331,7 +332,10 @@ class KafkaClient {
         /**
          * Consume messages from custom topic
          */
-        this.consumeCustom = async (topicName, handler, fromBeginning = false) => {
+        this.consumeCustom = async (topicName, handler, fromBeginning = false, onPoison) => {
+            const poisonHandler = onPoison === true
+                ? this.defaultPoisonHandler(topicName)
+                : onPoison || undefined;
             const topic = await this.getCustomTopic(topicName);
             const groupId = `${this.environment.valueOf()}_${topicName}_${this.serviceIdentity.identityName}`;
             try {
@@ -345,7 +349,7 @@ class KafkaClient {
                 await consumer.subscribe({ topic, fromBeginning });
                 await consumer.run({
                     autoCommit: true,
-                    eachMessage: async ({ message }) => {
+                    eachMessage: async ({ message, partition }) => {
                         try {
                             messageConsumedCounter.labels({ topic: topicName, groupId }).inc();
                             // Decrypt if secret was provided
@@ -359,21 +363,35 @@ class KafkaClient {
                                 value,
                                 timestamp: message.timestamp,
                             }, 'Custom topic record received');
-                            try {
-                                await handler({ ...message, value });
-                            }
-                            catch (error) {
-                                throw (0, utils_1.logAndGetError)(this.logger, `Could not consume message for custom topic=${topicName}`, error);
-                            }
+                            await handler({ ...message, value });
                             messageAcknowledgedCounter
                                 .labels({ topic: topicName, groupId })
                                 .inc();
                         }
                         catch (error) {
-                            messageConsumedErrorCounter
-                                .labels({ topic: topicName, groupId })
-                                .inc();
-                            throw error;
+                            if (poisonHandler) {
+                                this.logger.warn(error, 'Poison message detected, forwarding to DLQ');
+                                await poisonHandler({
+                                    error,
+                                    decryptFailed: false,
+                                    message,
+                                    partition,
+                                    primaryTopicFullName: topic,
+                                    rawPayloadUtf8: message.value
+                                        ? Buffer.from(message.value).toString('utf8')
+                                        : '',
+                                });
+                                messageAcknowledgedCounter
+                                    .labels({ topic: topicName, groupId })
+                                    .inc();
+                                return;
+                            }
+                            else {
+                                messageConsumedErrorCounter
+                                    .labels({ topic: topicName, groupId })
+                                    .inc();
+                                throw (0, utils_1.logAndGetError)(this.logger, `Could not consume message for custom topic=${topicName}`, error);
+                            }
                         }
                     },
                 });
@@ -383,6 +401,10 @@ class KafkaClient {
                 throw (0, utils_1.logAndGetError)(this.logger, `Could not create consumer for custom topic=${topicName}`, error);
             }
         };
+        this.defaultPoisonHandler = (topicName) => (0, defaultPoisonHandler_1.createDefaultPoisonHandler)(topicName, {
+            logger: this.logger,
+            produceCustom: (t, k, v) => this.produceCustom(t, k, v),
+        });
         this.encryptCustomValue = async (secret, value) => {
             const jwe = await new jose.CompactEncrypt(new util_1.TextEncoder().encode(value));
             jwe.setProtectedHeader({ alg: 'A256GCMKW', enc: 'A256GCM' });

package/dist/lib/kafka/types.d.ts

@@ -1,6 +1,6 @@
 import { KafkaTopic, MessageFormats } from './events';
 import { Environment, Service } from '../serviceIdentity';
-import { KafkaMessage } from 'kafkajs';
+import type { KafkaMessage } from 'kafkajs';
 type KafkaEvent<T extends KafkaTopic> = {
     id: string;
     type: 'create' | 'update' | 'soft-destroy' | 'destroy';
@@ -31,4 +31,13 @@ type CustomTopicConfig = {
     issuer: Service;
     secret?: string;
 };
-export type { KafkaEvent, GenericKafkaEvent, KafkaConfiguration, EventPayloadHandler, GenericEventPayloadHandler, CustomTopicConfig, };
+type CustomTopicPoisonContext = {
+    error: unknown;
+    decryptFailed: boolean;
+    message: KafkaMessage;
+    partition: number;
+    primaryTopicFullName: string;
+    rawPayloadUtf8: string;
+};
+type CustomTopicPoisonHandler = (ctx: CustomTopicPoisonContext) => Promise<void>;
+export type { KafkaEvent, GenericKafkaEvent, KafkaConfiguration, EventPayloadHandler, GenericEventPayloadHandler, CustomTopicConfig, CustomTopicPoisonContext, CustomTopicPoisonHandler, };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lucaapp/service-utils",
-  "version": "5.10.0",
+  "version": "5.11.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [
@@ -31,7 +31,7 @@
     "@types/response-time": "^2.3.9",
     "@types/swagger-ui-express": "4.1.8",
     "@types/validator": "13.7.1",
-    "axios": "^1.15.2",
+    "axios": "1.16.0",
     "axios-rate-limit": "^1.4.0",
     "axios-retry": "^4.5.0",
     "body-parser": "1.20.4",
@@ -97,7 +97,8 @@
     "node-forge": "1.4.0",
     "uuid": "^14.0.0",
     "follow-redirects": "^1.16.0",
-    "axios": "^1.15.2",
-    "path-to-regexp": "0.1.13"
+    "axios": "1.16.0",
+    "path-to-regexp": "0.1.13",
+    "ip-address": "10.1.1"
   }
 }