kafka-ts 0.0.4 → 0.0.5

package/README.md

@@ -85,6 +85,37 @@ See the [examples](./examples) for more detailed examples.
 
 By default KafkaTS logs out using a JSON logger. This can be globally replaced by calling setLogger method (see [src/utils/logger.ts](./src/utils/logger.ts))
 
+#### Retries
+
+By default KafkaTS retries onBatch and onMessage using an exponential backoff strategy (see [src/utils/retrier.ts](./src/utils/retrier.ts)). In case of failure, processed message offsets are committed, and the consumer is restarted.
+
+In case you want to skip failed messages or implement a DLQ-like mechanism, you can overwrite `retrier` on `startConsumer()` and execute your own logic `onFailure`.
+
+Example if you simply want to skip the failing messages:
+
+```typescript
+await kafka.startConsumer({
+    // ...
+    retrier: createExponentialBackoffRetrier({ onFailure: () => {} }),
+});
+```
+
+#### Concurrency control
+
+Depending on the use case, you might want to control `concurrency` and `batchGranularity`.
+
+When subscribing to a topic, the consumer group leader will distribute all subscribed topic partitions to consumers within the group. Each consumer will then fetch messages only from partitions assigned to them. 
+
+`batchGranularity` controls how messages are split into batches from a fetch response:
+
+- **broker** - (default) all messages received from a single kafka broker will be included in a single batch.
+- **topic** - all messages received from a single broker and topic will be included in a single batch.
+- **partition** - a batch will only include messages from a single partition.
+
+After each batch is processed, the consumer will commit offsets for the processed messages. The more granual the batch is, the more often offsets are committed.
+
+`concurrency` controls how many aforementioned batches are processed concurrently. 
+
 ## Motivation
 
 The existing low-level libraries (e.g. node-rdkafka) are bindings on librdkafka, which doesn't give enough control over the consumer logic.
@@ -132,7 +163,7 @@ Custom SASL mechanisms can be implemented following the `SASLProvider` interface
 | partitionMaxBytes      | number                                 | false    | 1_048_576                       | Maximum number of bytes to return per partition in the fetch response                                                                                                                                                                                                            |
 | allowTopicAutoCreation | boolean                                | false    | false                           | Allow kafka to auto-create topic when it doesn't exist                                                                                                                                                                                                                           |
 | fromBeginning          | boolean                                | false    | false                           | Start consuming from the beginning of the topic                                                                                                                                                                                                                                  |
-| batchGranularity       | BatchGranularity                       | false    | partition                       | Controls messages split from fetch response. Also controls how often offsets are committed. **onBatch** will include messages:<br/>- **partition** - from a single batch<br/>- **topic** - from all topic partitions<br/>- **broker** - from all assignned topics and partitions |
+| batchGranularity       | BatchGranularity                       | false    | broker                          | Controls messages split from fetch response. Also controls how often offsets are committed. **onBatch** will include messages:<br/>- **partition** - from a single batch<br/>- **topic** - from all topic partitions<br/>- **broker** - from all assignned topics and partitions |
 | concurrency            | number                                 | false    | 1                               | How many batches to process concurrently                                                                                                                                                                                                                                         |
 | onMessage              | (message: Message) => Promise<unknown> | true     |                                 | Callback executed on every message                                                                                                                                                                                                                                               |
 | onBatch                | (batch: Message[]) => Promise<unknown> | true     |                                 | Callback executed on every batch of messages (based on **batchGranuality**)                                                                                                                                                                                                      |

package/dist/consumer/consumer.d.ts

@@ -3,6 +3,7 @@ import EventEmitter from 'events';
 import { IsolationLevel } from '../api/fetch';
 import { Cluster } from '../cluster';
 import { Message } from '../types';
+import { Retrier } from '../utils/retrier';
 import { BatchGranularity } from './fetch-manager';
 export type ConsumerOptions = {
     topics: string[];
@@ -20,6 +21,7 @@ export type ConsumerOptions = {
     fromBeginning?: boolean;
     batchGranularity?: BatchGranularity;
     concurrency?: number;
+    retrier?: Retrier;
 } & ({
     onBatch: (messages: Required<Message>[]) => unknown;
 } | {

package/dist/consumer/consumer.js

@@ -19,6 +19,7 @@ const messages_to_topic_partition_leaders_1 = require("../distributors/messages-
 const delay_1 = require("../utils/delay");
 const error_1 = require("../utils/error");
 const logger_1 = require("../utils/logger");
+const retrier_1 = require("../utils/retrier");
 const tracer_1 = require("../utils/tracer");
 const consumer_group_1 = require("./consumer-group");
 const consumer_metadata_1 = require("./consumer-metadata");
@@ -50,8 +51,9 @@ class Consumer extends events_1.default {
             isolationLevel: options.isolationLevel ?? 0 /* IsolationLevel.READ_UNCOMMITTED */,
             allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
             fromBeginning: options.fromBeginning ?? false,
-            batchGranularity: options.batchGranularity ?? 'partition',
+            batchGranularity: options.batchGranularity ?? 'broker',
             concurrency: options.concurrency ?? 1,
+            retrier: options.retrier ?? retrier_1.defaultRetrier,
         };
         this.metadata = new consumer_metadata_1.ConsumerMetadata({ cluster: this.cluster });
         this.offsetManager = new offset_manager_1.OffsetManager({
@@ -84,7 +86,7 @@ class Consumer extends events_1.default {
             await this.consumerGroup?.join();
         }
         catch (error) {
-            logger_1.log.warn('Failed to start consumer', error);
+            logger_1.log.error('Failed to start consumer', error);
             logger_1.log.debug(`Restarting consumer in 1 second...`);
             await (0, delay_1.delay)(1000);
             if (this.stopHook)
@@ -152,7 +154,9 @@ class Consumer extends events_1.default {
                     break;
                 }
                 logger_1.log.error(error.message, error);
-                this.close();
+                logger_1.log.debug(`Restarting consumer in 1 second...`);
+                await (0, delay_1.delay)(1000);
+                this.close().then(() => this.start());
                 break;
             }
         }
@@ -160,24 +164,31 @@ class Consumer extends events_1.default {
     }
     async process(messages) {
         const { options } = this;
+        const { retrier } = options;
         const topicPartitions = {};
         for (const { topic, partition } of messages) {
             topicPartitions[topic] ??= new Set();
             topicPartitions[topic].add(partition);
         }
+        const commit = async () => {
+            await this.consumerGroup?.offsetCommit(topicPartitions);
+            this.offsetManager.flush(topicPartitions);
+        };
         if ('onBatch' in options) {
-            await options.onBatch(messages);
+            await retrier(() => options.onBatch(messages));
             messages.forEach(({ topic, partition, offset }) => this.offsetManager.resolve(topic, partition, offset + 1n));
         }
         else if ('onMessage' in options) {
             for (const message of messages) {
-                await options.onMessage(message);
+                await retrier(() => options.onMessage(message)).catch(async (error) => {
+                    await commit().catch();
+                    throw error;
+                });
                 const { topic, partition, offset } = message;
                 this.offsetManager.resolve(topic, partition, offset + 1n);
             }
         }
-        await this.consumerGroup?.offsetCommit(topicPartitions);
-        this.offsetManager.flush(topicPartitions);
+        await commit();
     }
     fetch(nodeId, assignment) {
         const { rackId, maxWaitMs, minBytes, maxBytes, partitionMaxBytes, isolationLevel } = this.options;

package/dist/index.d.ts

@@ -6,4 +6,5 @@ export * from './distributors/partitioner';
 export * from './types';
 export * from './utils/error';
 export * from './utils/logger';
+export * from './utils/retrier';
 export { Tracer, setTracer } from './utils/tracer';

package/dist/index.js

@@ -22,5 +22,6 @@ __exportStar(require("./distributors/partitioner"), exports);
 __exportStar(require("./types"), exports);
 __exportStar(require("./utils/error"), exports);
 __exportStar(require("./utils/logger"), exports);
+__exportStar(require("./utils/retrier"), exports);
 var tracer_1 = require("./utils/tracer");
 Object.defineProperty(exports, "setTracer", { enumerable: true, get: function () { return tracer_1.setTracer; } });

package/dist/utils/retrier.d.ts

@@ -1,10 +1,9 @@
 export type Retrier = (func: () => unknown) => Promise<void>;
-export declare const createExponentialBackoffRetrier: (options: {
-    onFailure?: (error: unknown) => Promise<void>;
-    maxRetries?: number;
+export declare const createExponentialBackoffRetrier: ({ retries, initialDelayMs, maxDelayMs, multiplier, onFailure, }?: {
+    retries?: number;
     initialDelayMs?: number;
     maxDelayMs?: number;
     multiplier?: number;
-    retry?: number;
+    onFailure?: (error: unknown) => unknown;
 }) => Retrier;
 export declare const defaultRetrier: Retrier;

package/dist/utils/retrier.js

@@ -2,21 +2,26 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.defaultRetrier = exports.createExponentialBackoffRetrier = void 0;
 const delay_1 = require("./delay");
-const createExponentialBackoffRetrier = (options) => async (func) => {
-    try {
-        await func();
-    }
-    catch (error) {
-        const { retry = 0, maxRetries = 3, onFailure = (error) => {
-            throw error;
-        }, initialDelayMs = 100, maxDelayMs = 3000, multiplier = 2, } = options;
-        const isMaxRetriesExceeded = retry > maxRetries;
-        if (isMaxRetriesExceeded)
-            return onFailure(error);
-        const delayMs = Math.min(maxDelayMs, initialDelayMs * multiplier ** retry);
+const createExponentialBackoffRetrier = ({ retries = 5, initialDelayMs = 100, maxDelayMs = 3000, multiplier = 2, onFailure = (error) => {
+    throw error;
+}, } = {}) => async (func) => {
+    let retriesLeft = retries;
+    let delayMs = initialDelayMs;
+    let lastError;
+    while (true) {
+        try {
+            await func();
+            return;
+        }
+        catch (error) {
+            lastError = error;
+        }
+        if (--retriesLeft < 1)
+            break;
         await (0, delay_1.delay)(delayMs);
-        return (0, exports.createExponentialBackoffRetrier)({ ...options, retry: retry + 1 })(func);
+        delayMs = Math.min(maxDelayMs, delayMs * multiplier);
     }
+    await onFailure(lastError);
 };
 exports.createExponentialBackoffRetrier = createExponentialBackoffRetrier;
-exports.defaultRetrier = (0, exports.createExponentialBackoffRetrier)({});
+exports.defaultRetrier = (0, exports.createExponentialBackoffRetrier)();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.4",
+    "version": "0.0.5",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",