kafka-ts 0.0.6 → 0.0.8

package/dist/api/index.d.ts

@@ -446,7 +446,7 @@ export declare const API: {
         errorCode: number;
         protocolType: string | null;
         protocolName: string | null;
-        assignments: string;
+        assignments: import("./sync-group").Assignment;
         _tag2: void;
     }>;
 };

package/dist/api/sync-group.d.ts

@@ -19,6 +19,6 @@ export declare const SYNC_GROUP: import("../utils/api").Api<{
     errorCode: number;
     protocolType: string | null;
     protocolName: string | null;
-    assignments: string;
+    assignments: Assignment;
     _tag2: void;
 }>;

package/dist/api/sync-group.js

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SYNC_GROUP = void 0;
 const api_1 = require("../utils/api");
+const decoder_1 = require("../utils/decoder");
+const encoder_1 = require("../utils/encoder");
 const error_1 = require("../utils/error");
 exports.SYNC_GROUP = (0, api_1.createApi)({
     apiKey: 14,
@@ -16,7 +18,7 @@ exports.SYNC_GROUP = (0, api_1.createApi)({
         .writeCompactString(data.protocolName)
         .writeCompactArray(data.assignments, (encoder, assignment) => encoder
         .writeCompactString(assignment.memberId)
-        .writeCompactString(JSON.stringify(assignment.assignment))
+        .writeCompactBytes(encodeAssignment(assignment.assignment))
         .writeUVarInt(0))
         .writeUVarInt(0),
     response: (decoder) => {
@@ -26,7 +28,7 @@ exports.SYNC_GROUP = (0, api_1.createApi)({
             errorCode: decoder.readInt16(),
             protocolType: decoder.readCompactString(),
             protocolName: decoder.readCompactString(),
-            assignments: decoder.readCompactString(),
+            assignments: decodeAssignment(decoder.readCompactBytes()),
             _tag2: decoder.readTagBuffer(),
         };
         if (result.errorCode)
@@ -34,3 +36,20 @@ exports.SYNC_GROUP = (0, api_1.createApi)({
         return result;
     },
 });
+const encodeAssignment = (data) => new encoder_1.Encoder()
+    .writeInt16(0)
+    .writeArray(Object.entries(data), (encoder, [topic, partitions]) => encoder.writeString(topic).writeArray(partitions, (encoder, partition) => encoder.writeInt32(partition)))
+    .writeBytes(Buffer.alloc(0))
+    .value();
+const decodeAssignment = (data) => {
+    const decoder = new decoder_1.Decoder(data);
+    const result = {
+        version: decoder.readInt16(),
+        assignment: decoder.readArray((decoder) => ({
+            topic: decoder.readString(),
+            partitions: decoder.readArray((decoder) => decoder.readInt32()),
+        })),
+        userData: decoder.readBytes(),
+    };
+    return Object.fromEntries(result.assignment.map(({ topic, partitions }) => [topic, partitions]));
+};

package/dist/consumer/consumer-group.d.ts

@@ -1,6 +1,5 @@
-/// <reference types="node" />
-import EventEmitter from 'events';
 import { Cluster } from '../cluster';
+import { Consumer } from './consumer';
 import { ConsumerMetadata } from './consumer-metadata';
 import { OffsetManager } from './offset-manager';
 type ConsumerGroupOptions = {
@@ -12,10 +11,9 @@ type ConsumerGroupOptions = {
     rebalanceTimeoutMs: number;
     metadata: ConsumerMetadata;
     offsetManager: OffsetManager;
+    consumer: Consumer;
 };
-export declare class ConsumerGroup extends EventEmitter<{
-    offsetCommit: [];
-}> {
+export declare class ConsumerGroup {
     private options;
     private coordinatorId;
     private memberId;
@@ -25,11 +23,11 @@ export declare class ConsumerGroup extends EventEmitter<{
     private heartbeatInterval;
     private heartbeatError;
     constructor(options: ConsumerGroupOptions);
+    init(): Promise<void>;
     join(): Promise<void>;
     private startHeartbeater;
     private stopHeartbeater;
     handleLastHeartbeat(): void;
-    resetHeartbeat(): void;
     private findCoordinator;
     private joinGroup;
     private syncGroup;

package/dist/consumer/consumer-group.js

@@ -8,17 +8,13 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ConsumerGroup = void 0;
-const events_1 = __importDefault(require("events"));
 const api_1 = require("../api");
 const find_coordinator_1 = require("../api/find-coordinator");
 const tracer_1 = require("../utils/tracer");
 const trace = (0, tracer_1.createTracer)('ConsumerGroup');
-class ConsumerGroup extends events_1.default {
+class ConsumerGroup {
     options;
     coordinatorId = -1;
     memberId = '';
@@ -28,19 +24,22 @@ class ConsumerGroup extends events_1.default {
     heartbeatInterval = null;
     heartbeatError = null;
     constructor(options) {
-        super();
         this.options = options;
     }
-    async join() {
+    async init() {
         await this.findCoordinator();
         await this.options.cluster.setSeedBroker(this.coordinatorId);
         this.memberId = '';
+    }
+    async join() {
         await this.joinGroup();
         await this.syncGroup();
         await this.offsetFetch();
         this.startHeartbeater();
     }
     async startHeartbeater() {
+        this.stopHeartbeater();
+        this.heartbeatError = null;
         this.heartbeatInterval = setInterval(async () => {
             try {
                 await this.heartbeat();
@@ -61,9 +60,6 @@ class ConsumerGroup extends events_1.default {
             throw this.heartbeatError;
         }
     }
-    resetHeartbeat() {
-        this.heartbeatError = null;
-    }
     async findCoordinator() {
         const { coordinators } = await this.options.cluster.sendRequest(api_1.API.FIND_COORDINATOR, {
             keyType: find_coordinator_1.KEY_TYPE.GROUP,
@@ -121,7 +117,7 @@ class ConsumerGroup extends events_1.default {
             protocolName: 'RoundRobinAssigner',
             assignments,
         });
-        metadata.setAssignment(JSON.parse(response.assignments || '{}'));
+        metadata.setAssignment(response.assignments);
     }
     async offsetFetch() {
         const { cluster, groupId, topics, metadata, offsetManager } = this.options;
@@ -155,7 +151,7 @@ class ConsumerGroup extends events_1.default {
         offsetManager.flush(topicPartitions);
     }
     async offsetCommit(topicPartitions) {
-        const { cluster, groupId, groupInstanceId, offsetManager } = this.options;
+        const { cluster, groupId, groupInstanceId, offsetManager, consumer } = this.options;
         const request = {
             groupId,
             groupInstanceId,
@@ -175,16 +171,17 @@ class ConsumerGroup extends events_1.default {
             return;
         }
         await cluster.sendRequest(api_1.API.OFFSET_COMMIT, request);
-        this.emit('offsetCommit');
+        consumer.emit('offsetCommit');
     }
     async heartbeat() {
-        const { cluster, groupId, groupInstanceId } = this.options;
+        const { cluster, groupId, groupInstanceId, consumer } = this.options;
         await cluster.sendRequest(api_1.API.HEARTBEAT, {
             groupId,
             groupInstanceId,
             memberId: this.memberId,
             generationId: this.generationId,
         });
+        consumer.emit('heartbeat');
     }
     async leaveGroup() {
         if (this.coordinatorId === -1) {
@@ -207,6 +204,12 @@ class ConsumerGroup extends events_1.default {
     }
 }
 exports.ConsumerGroup = ConsumerGroup;
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], ConsumerGroup.prototype, "init", null);
 __decorate([
     trace(),
     __metadata("design:type", Function),

package/dist/consumer/consumer.d.ts

@@ -29,6 +29,7 @@ export type ConsumerOptions = {
 });
 export declare class Consumer extends EventEmitter<{
     offsetCommit: [];
+    heartbeat: [];
 }> {
     private cluster;
     private options;

package/dist/consumer/consumer.js

@@ -71,9 +71,9 @@ class Consumer extends events_1.default {
                 rebalanceTimeoutMs: this.options.rebalanceTimeoutMs,
                 metadata: this.metadata,
                 offsetManager: this.offsetManager,
+                consumer: this,
             })
             : undefined;
-        this.consumerGroup?.on('offsetCommit', () => this.emit('offsetCommit'));
     }
     async start() {
         const { topics, allowTopicAutoCreation, fromBeginning } = this.options;
@@ -83,7 +83,7 @@ class Consumer extends events_1.default {
             await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
             this.metadata.setAssignment(this.metadata.getTopicPartitions());
             await this.offsetManager.fetchOffsets({ fromBeginning });
-            await this.consumerGroup?.join();
+            await this.consumerGroup?.init();
         }
         catch (error) {
             logger_1.log.error('Failed to start consumer', error);
@@ -108,7 +108,7 @@ class Consumer extends events_1.default {
     async startFetchManager() {
         const { batchGranularity, concurrency } = this.options;
         while (!this.stopHook) {
-            this.consumerGroup?.resetHeartbeat();
+            await this.consumerGroup?.join();
             // TODO: If leader is not available, find another read replica
             const nodeAssignments = Object.entries((0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(Object.entries(this.metadata.getAssignment()).flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition }))), this.metadata.getTopicPartitionLeaderIds())).map(([nodeId, assignment]) => ({
                 nodeId: parseInt(nodeId),

package/dist/consumer/fetch-manager.js

@@ -25,11 +25,10 @@ class FetchManager {
     fetcherCallbacks = {};
     constructor(options) {
         this.options = options;
-        const { fetch, process, consumerGroup, nodeAssignments, concurrency } = this.options;
+        const { fetch, process, nodeAssignments, concurrency } = this.options;
         this.fetchers = nodeAssignments.map(({ nodeId, assignment }, index) => new fetcher_1.Fetcher(index, {
             nodeId,
             assignment,
-            consumerGroup,
             fetch,
             onResponse: this.onResponse.bind(this),
         }));
@@ -64,6 +63,8 @@ class FetchManager {
         if (!this.isRunning) {
             return [];
         }
+        const { consumerGroup } = this.options;
+        consumerGroup?.handleLastHeartbeat();
         const batch = this.queue.shift();
         if (!batch) {
             // wait until new data is available or fetch manager is requested to stop
@@ -80,7 +81,8 @@ class FetchManager {
         return batch;
     }
     async onResponse(fetcherId, response) {
-        const { metadata, batchGranularity } = this.options;
+        const { metadata, batchGranularity, consumerGroup } = this.options;
+        consumerGroup?.handleLastHeartbeat();
         const batches = fetchResponseToBatches(response, batchGranularity, metadata);
         if (!batches.length) {
             return;
@@ -91,6 +93,7 @@ class FetchManager {
             this.queue.push(...batches, { kind: 'checkpoint', fetcherId });
             this.pollQueue?.shift()?.();
         });
+        consumerGroup?.handleLastHeartbeat();
     }
 }
 exports.FetchManager = FetchManager;

package/dist/consumer/fetcher.d.ts

@@ -3,11 +3,9 @@
 import { EventEmitter } from 'stream';
 import { FetchResponse } from '../api/fetch';
 import { Assignment } from '../api/sync-group';
-import { ConsumerGroup } from './consumer-group';
 type FetcherOptions = {
     nodeId: number;
     assignment: Assignment;
-    consumerGroup?: ConsumerGroup;
     fetch: (nodeId: number, assignment: Assignment) => Promise<FetchResponse>;
     onResponse: (fetcherId: number, response: FetchResponse) => Promise<void>;
 };

package/dist/consumer/fetcher.js

@@ -35,14 +35,12 @@ class Fetcher extends stream_1.EventEmitter {
         }
     }
     async step() {
-        const { nodeId, assignment, consumerGroup, fetch, onResponse } = this.options;
+        const { nodeId, assignment, fetch, onResponse } = this.options;
         const response = await fetch(nodeId, assignment);
         if (!this.isRunning) {
             return;
         }
-        consumerGroup?.handleLastHeartbeat();
         await onResponse(this.fetcherId, response);
-        consumerGroup?.handleLastHeartbeat();
     }
     async stop() {
         if (!this.isRunning) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.6",
+    "version": "0.0.8",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",
@@ -9,7 +9,8 @@
         "url": "https://github.com/priitkaard/kafka-ts.git"
     },
     "scripts": {
-        "start": "docker-compose down && KAFKA_VERSION=3.7.1 docker-compose up -d && sleep 5 && bash ./scripts/create-scram-user.sh",
+        "up": "npm run down && KAFKA_VERSION=3.7.1 docker-compose up -d && sleep 5 && bash ./scripts/create-scram-user.sh",
+        "down": "KAFKA_VERSION=3.7.1 docker-compose down",
         "version:prerelease": "npm version prerelease",
         "version:patch": "npm version patch",
         "format": "prettier --write .",