kafka-ts 1.2.1 → 1.3.1-beta.0

package/README.md

@@ -148,6 +148,7 @@ The existing high-level libraries (e.g. kafkajs) are missing a few crucial featu
 -   PLAIN: `saslPlain({ username, password })`
 -   SCRAM-SHA-256: `saslScramSha256({ username, password })`
 -   SCRAM-SHA-512: `saslScramSha512({ username, password })`
+-   OAUTHBEARER: `oAuthBearer(oAuthAuthenticator({ endpoint, clientId, clientSecret }))`
 
 Custom SASL mechanisms can be implemented following the `SASLProvider` interface. See [src/auth](./src/auth) for examples.
 

package/dist/auth/index.d.ts

@@ -1,2 +1,3 @@
+export { oAuthAuthenticator, oAuthBearer } from './oauthbearer';
 export { saslPlain } from './plain';
 export { saslScramSha256, saslScramSha512 } from './scram';

package/dist/auth/index.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.saslScramSha512 = exports.saslScramSha256 = exports.saslPlain = void 0;
+exports.saslScramSha512 = exports.saslScramSha256 = exports.saslPlain = exports.oAuthBearer = exports.oAuthAuthenticator = void 0;
+var oauthbearer_1 = require("./oauthbearer");
+Object.defineProperty(exports, "oAuthAuthenticator", { enumerable: true, get: function () { return oauthbearer_1.oAuthAuthenticator; } });
+Object.defineProperty(exports, "oAuthBearer", { enumerable: true, get: function () { return oauthbearer_1.oAuthBearer; } });
 var plain_1 = require("./plain");
 Object.defineProperty(exports, "saslPlain", { enumerable: true, get: function () { return plain_1.saslPlain; } });
 var scram_1 = require("./scram");

package/dist/auth/oauthbearer.d.ts

@@ -0,0 +1,16 @@
+import { SASLProvider } from '../broker';
+export declare const oAuthBearer: (getToken: () => Promise<{
+    access_token: string;
+}>) => SASLProvider;
+export declare const oAuthAuthenticator: ({ endpoint, clientId, clientSecret, refreshThresholdSeconds, }: {
+    endpoint: string;
+    clientId: string;
+    clientSecret: string;
+    refreshThresholdSeconds?: number;
+}) => () => Promise<TokenResponse>;
+type TokenResponse = {
+    access_token: string;
+    refresh_token: string;
+    expires_in: number;
+};
+export {};

package/dist/auth/oauthbearer.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.oAuthAuthenticator = exports.oAuthBearer = void 0;
+const api_1 = require("../api");
+const retry_1 = require("../utils/retry");
+const oAuthBearer = (getToken) => {
+    return {
+        mechanism: 'OAUTHBEARER',
+        authenticate: async ({ sendRequest }) => {
+            const { access_token: accessToken } = await getToken();
+            const sep = String.fromCharCode(1);
+            const authBytes = `n,,${sep}auth=Bearer ${accessToken}${sep}${sep}`;
+            await sendRequest(api_1.API.SASL_AUTHENTICATE, { authBytes: Buffer.from(authBytes) });
+        },
+    };
+};
+exports.oAuthBearer = oAuthBearer;
+const oAuthAuthenticator = ({ endpoint, clientId, clientSecret, refreshThresholdSeconds = 15, }) => {
+    let tokenPromise = createToken(endpoint, {
+        grant_type: 'client_credentials',
+        client_id: clientId,
+        client_secret: clientSecret,
+    });
+    const scheduleRefresh = () => {
+        tokenPromise.then((token) => {
+            const refreshInMs = (token.expires_in - refreshThresholdSeconds) * 1000;
+            setTimeout(() => {
+                tokenPromise = createToken(endpoint, {
+                    grant_type: 'refresh_token',
+                    client_id: clientId,
+                    client_secret: clientSecret,
+                    refresh_token: token.refresh_token,
+                });
+                scheduleRefresh();
+            }, refreshInMs);
+        });
+    };
+    scheduleRefresh();
+    return () => tokenPromise;
+};
+exports.oAuthAuthenticator = oAuthAuthenticator;
+const createToken = async (endpoint, body) => {
+    return (0, retry_1.withRetry)((0, retry_1.exponentialBackoff)(100), 5)(async () => {
+        const response = await fetch(endpoint, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+            body: new URLSearchParams(body),
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to obtain OAuth token: ${await response.text()}`);
+        }
+        return response.json();
+    });
+};

package/dist/consumer/consumer-group.js

@@ -48,6 +48,9 @@ class ConsumerGroup {
             }
             catch (error) {
                 this.heartbeatError = error;
+                if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.REBALANCE_IN_PROGRESS) {
+                    this.options.consumer.emit('rebalanceInProgress');
+                }
             }
         }, 5000);
     }

package/dist/consumer/consumer.d.ts

@@ -21,11 +21,13 @@ export type ConsumerOptions = {
     retrier?: Retrier;
     onBatch: (messages: Required<Message>[], context: {
         resolveOffset: (message: Pick<Required<Message>, 'topic' | 'partition' | 'offset'>) => void;
+        abortSignal: AbortSignal;
     }) => unknown;
 };
 export declare class Consumer extends EventEmitter<{
     offsetCommit: [];
     heartbeat: [];
+    rebalanceInProgress: [];
 }> {
     private cluster;
     private options;

package/dist/consumer/consumer.js

@@ -75,6 +75,7 @@ class Consumer extends events_1.default {
                 consumer: this,
             })
             : undefined;
+        this.setMaxListeners(Infinity);
     }
     async start() {
         this.stopHook = undefined;
@@ -191,26 +192,34 @@ class Consumer extends events_1.default {
         if (!messages.length) {
             return;
         }
+        const commitOffset = () => this.consumerGroup?.offsetCommit(topicPartitions).then(() => this.offsetManager.flush(topicPartitions));
         const resolveOffset = (message) => this.offsetManager.resolve(message.topic, message.partition, message.offset + 1n);
+        const abortController = new AbortController();
+        const onRebalance = () => {
+            abortController.abort();
+            commitOffset()?.catch();
+        };
+        this.once('rebalanceInProgress', onRebalance);
         try {
-            await retrier(() => options.onBatch(messages.filter((message) => !this.offsetManager.isResolved(message)), { resolveOffset }));
+            await retrier(() => options.onBatch(messages.filter((message) => !this.offsetManager.isResolved(message)), { resolveOffset, abortSignal: abortController.signal }));
         }
         catch (error) {
-            await this.consumerGroup
-                ?.offsetCommit(topicPartitions)
-                .then(() => this.offsetManager.flush(topicPartitions))
-                .catch();
+            await commitOffset()?.catch();
             throw error;
         }
-        response.responses.forEach(({ topicId, partitions }) => {
-            partitions.forEach(({ partitionIndex, records }) => {
-                records.forEach(({ baseOffset, lastOffsetDelta }) => {
-                    this.offsetManager.resolve(this.metadata.getTopicNameById(topicId), partitionIndex, baseOffset + BigInt(lastOffsetDelta) + 1n);
+        finally {
+            this.off('rebalanceInProgress', onRebalance);
+        }
+        if (!abortController.signal.aborted) {
+            response.responses.forEach(({ topicId, partitions }) => {
+                partitions.forEach(({ partitionIndex, records }) => {
+                    records.forEach(({ baseOffset, lastOffsetDelta }) => {
+                        this.offsetManager.resolve(this.metadata.getTopicNameById(topicId), partitionIndex, baseOffset + BigInt(lastOffsetDelta) + 1n);
+                    });
                 });
             });
-        });
-        await this.consumerGroup?.offsetCommit(topicPartitions);
-        this.offsetManager.flush(topicPartitions);
+        }
+        await commitOffset();
     }
     async fetch(nodeId, assignment) {
         return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {

package/dist/utils/retry.d.ts

@@ -1 +1,2 @@
-export declare const withRetry: (handleError: (error: unknown) => Promise<void>) => <T>(func: () => Promise<T>) => Promise<T>;
+export declare const withRetry: (handleError: (error: unknown, retry: number) => Promise<void>, maxRetries?: number) => <T>(func: () => Promise<T>) => Promise<T>;
+export declare const exponentialBackoff: (initialDelayMs: number, maxDelayMs?: number) => (_: unknown, retry: number) => Promise<void>;

package/dist/utils/retry.js

@@ -1,15 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.withRetry = void 0;
+exports.exponentialBackoff = exports.withRetry = void 0;
+const delay_1 = require("./delay");
 const logger_1 = require("./logger");
-const withRetry = (handleError) => async (func) => {
+const withRetry = (handleError, maxRetries = 15) => async (func) => {
     let lastError;
-    for (let i = 0; i < 15; i++) {
+    for (let i = 0; i < maxRetries; i++) {
         try {
             return await func();
         }
         catch (error) {
-            await handleError(error);
+            await handleError(error, i + 1);
             lastError = error;
         }
     }
@@ -17,3 +18,8 @@ const withRetry = (handleError) => async (func) => {
     throw lastError;
 };
 exports.withRetry = withRetry;
+const exponentialBackoff = (initialDelayMs, maxDelayMs = 5_000) => async (_, retry) => {
+    const delayMs = Math.min(maxDelayMs, initialDelayMs * 2 ** retry);
+    await (0, delay_1.delay)(delayMs);
+};
+exports.exponentialBackoff = exponentialBackoff;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.2.1",
+    "version": "1.3.1-beta.0",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",
@@ -13,6 +13,7 @@
         "down": "KAFKA_VERSION=4.0.0 docker-compose down",
         "version:prerelease": "npm version prerelease --preid=beta",
         "version:patch": "npm version patch",
+        "version:minor": "npm version minor",
         "version:major": "npm version major",
         "format": "prettier --write .",
         "build": "tsc",