@lucaapp/service-utils 5.41.0 → 5.42.1

package/dist/lib/kafka/kafkaClient.d.ts

@@ -48,6 +48,8 @@ declare class KafkaClient {
      */
     consumeCustom: <T = unknown>(topicName: string, handler: GenericEventPayloadHandler<T>, fromBeginning?: boolean, onPoison?: boolean | CustomTopicPoisonHandler) => Promise<Consumer>;
     private runCustomConsumer;
+    private processBatchByKey;
+    private processCustomMessage;
     private defaultPoisonHandler;
     private encryptCustomValue;
     private decryptCustomValue;

package/dist/lib/kafka/kafkaClient.js

@@ -45,6 +45,8 @@ const utils_1 = require("../../utils/utils");
 const topicLifecycle_1 = require("./topicLifecycle");
 const metrics_1 = require("./metrics");
 const KEY_ALG = 'ES256';
+// kafkajs commitOffsetsIfNecessary is a no-op without an interval or threshold.
+const BATCH_AUTO_COMMIT_INTERVAL_MS = 5_000;
 const getIssuer = (kafkaTopic) => {
     return events_1.MessageIssuer[kafkaTopic].valueOf();
 };
@@ -278,6 +280,11 @@ class KafkaClient {
          * This allows services to define their own topics without modifying service-utils
          */
         this.registerCustomTopic = (config) => {
+            const { keyConcurrency } = config;
+            if (keyConcurrency !== undefined &&
+                !(Number.isInteger(keyConcurrency) && keyConcurrency >= 1)) {
+                throw (0, utils_1.logAndGetError)(this.logger, `Invalid keyConcurrency=${keyConcurrency} for custom topic=${config.topic}. Expected an integer >= 1.`);
+            }
             this.customTopics.set(config.topic, config);
             this.logger.info({ topic: config.topic, issuer: config.issuer }, 'Registered custom Kafka topic');
         };
@@ -362,60 +369,127 @@ class KafkaClient {
         };
         this.runCustomConsumer = async (consumer, topic, fromBeginning, topicName, handler, groupId, poisonHandler) => {
             await consumer.subscribe({ topic, fromBeginning });
+            const processMessage = (message, partition) => this.processCustomMessage(message, partition, topic, topicName, handler, groupId, poisonHandler);
+            const keyConcurrency = this.customTopics.get(topicName)?.keyConcurrency;
+            if (!keyConcurrency) {
+                await consumer.run({
+                    autoCommit: true,
+                    eachMessage: ({ message, partition }) => processMessage(message, partition),
+                });
+                return;
+            }
             await consumer.run({
                 autoCommit: true,
-                eachMessage: async ({ message, partition }) => {
-                    if (this.isForeignEvent(message.headers)) {
-                        metrics_1.messageSkippedForeignEnvCounter
-                            .labels({ topic: topicName, groupId })
-                            .inc();
+                autoCommitInterval: BATCH_AUTO_COMMIT_INTERVAL_MS,
+                eachBatchAutoResolve: false,
+                eachBatch: payload => this.processBatchByKey(payload, keyConcurrency, processMessage),
+            });
+        };
+        // Same key runs in offset order, different keys run in parallel; only the contiguous done prefix gets resolved.
+        this.processBatchByKey = async ({ batch, resolveOffset, heartbeat, isRunning, isStale, commitOffsetsIfNecessary, }, keyConcurrency, processMessage) => {
+            const { messages, partition } = batch;
+            const groups = new Map();
+            messages.forEach((message, index) => {
+                const key = message.key?.length ? message.key.toString() : null;
+                const group = groups.get(key);
+                if (group) {
+                    group.push(index);
+                }
+                else {
+                    groups.set(key, [index]);
+                }
+            });
+            const queue = [...groups.values()];
+            const done = new Array(messages.length).fill(false);
+            let watermark = 0;
+            let failed = false;
+            let firstError;
+            const canContinue = () => !failed && isRunning() && !isStale();
+            const advance = async () => {
+                const before = watermark;
+                while (watermark < messages.length && done[watermark]) {
+                    watermark += 1;
+                }
+                if (watermark > before && !isStale()) {
+                    resolveOffset(messages[watermark - 1].offset);
+                    await commitOffsetsIfNecessary();
+                }
+            };
+            const worker = async () => {
+                while (canContinue()) {
+                    const group = queue.shift();
+                    if (!group) {
                         return;
                     }
-                    try {
-                        metrics_1.messageConsumedCounter.labels({ topic: topicName, groupId }).inc();
-                        // Decrypt if secret was provided
-                        const config = this.customTopics.get(topicName);
-                        const decryptedValue = config.secret && this.encryptionEnabled
-                            ? await this.decryptCustomValue(config.secret, message.value)
-                            : message.value;
-                        const value = this.parseCustomValue(decryptedValue);
-                        this.logger.debug({
-                            key: message.key?.toString(),
-                            value,
-                            timestamp: message.timestamp,
-                        }, 'Custom topic record received');
-                        await handler({ ...message, value });
-                        metrics_1.messageAcknowledgedCounter
-                            .labels({ topic: topicName, groupId })
-                            .inc();
-                    }
-                    catch (error) {
-                        if (poisonHandler) {
-                            this.logger.warn(error, 'Poison message detected, forwarding to DLQ');
-                            await poisonHandler({
-                                error,
-                                decryptFailed: false,
-                                message,
-                                partition,
-                                primaryTopicFullName: topic,
-                                rawPayloadUtf8: message.value
-                                    ? Buffer.from(message.value).toString('utf8')
-                                    : '',
-                            });
-                            metrics_1.messageAcknowledgedCounter
-                                .labels({ topic: topicName, groupId })
-                                .inc();
+                    for (const index of group) {
+                        if (!canContinue()) {
                             return;
                         }
-                        else {
-                            metrics_1.messageConsumedErrorCounter
-                                .labels({ topic: topicName, groupId })
-                                .inc();
-                            throw (0, utils_1.logAndGetError)(this.logger, `Could not consume message for custom topic=${topicName}`, error);
+                        try {
+                            await processMessage(messages[index], partition);
+                            done[index] = true;
+                            await advance();
+                            await heartbeat();
+                        }
+                        catch (error) {
+                            if (!failed) {
+                                failed = true;
+                                firstError = error;
+                            }
+                            return;
                         }
                     }
-                },
-            });
+                }
+            };
+            await Promise.all(Array.from({ length: Math.min(keyConcurrency, queue.length) }, worker));
+            if (failed) {
+                throw firstError;
+            }
+        };
+        this.processCustomMessage = async (message, partition, topic, topicName, handler, groupId, poisonHandler) => {
+            if (this.isForeignEvent(message.headers)) {
+                metrics_1.messageSkippedForeignEnvCounter
+                    .labels({ topic: topicName, groupId })
+                    .inc();
+                return;
+            }
+            try {
+                metrics_1.messageConsumedCounter.labels({ topic: topicName, groupId }).inc();
+                // Decrypt if secret was provided
+                const config = this.customTopics.get(topicName);
+                const decryptedValue = config.secret && this.encryptionEnabled
+                    ? await this.decryptCustomValue(config.secret, message.value)
+                    : message.value;
+                const value = this.parseCustomValue(decryptedValue);
+                this.logger.debug({
+                    key: message.key?.toString(),
+                    value,
+                    timestamp: message.timestamp,
+                }, 'Custom topic record received');
+                await handler({ ...message, value });
+                metrics_1.messageAcknowledgedCounter.labels({ topic: topicName, groupId }).inc();
+            }
+            catch (error) {
+                if (poisonHandler) {
+                    this.logger.warn(error, 'Poison message detected, forwarding to DLQ');
+                    await poisonHandler({
+                        error,
+                        decryptFailed: false,
+                        message,
+                        partition,
+                        primaryTopicFullName: topic,
+                        rawPayloadUtf8: message.value
+                            ? Buffer.from(message.value).toString('utf8')
+                            : '',
+                    });
+                    metrics_1.messageAcknowledgedCounter.labels({ topic: topicName, groupId }).inc();
+                    return;
+                }
+                else {
+                    metrics_1.messageConsumedErrorCounter.labels({ topic: topicName, groupId }).inc();
+                    throw (0, utils_1.logAndGetError)(this.logger, `Could not consume message for custom topic=${topicName}`, error);
+                }
+            }
         };
         this.defaultPoisonHandler = (topicName) => (0, defaultPoisonHandler_1.createDefaultPoisonHandler)(topicName, {
             logger: this.logger,

package/dist/lib/kafka/types.d.ts

@@ -30,6 +30,7 @@ type CustomTopicConfig = {
     topic: string;
     issuer: Service;
     secret?: string;
+    keyConcurrency?: number;
 };
 type CustomTopicPoisonContext = {
     error: unknown;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lucaapp/service-utils",
-  "version": "5.41.0",
+  "version": "5.42.1",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [
@@ -111,7 +111,7 @@
     "picomatch": "2.3.2",
     "node-forge": "1.4.0",
     "uuid": "^14.0.0",
-    "follow-redirects": "^1.16.0",
+    "follow-redirects": "^1.16.1",
     "axios": "1.20.0",
     "path-to-regexp": "0.1.13",
     "ip-address": "10.7.2",