kafka-ts 1.3.1-beta.1 → 1.3.1-beta.2

package/dist/api/fetch.js

@@ -4,11 +4,129 @@ exports.FETCH = void 0;
 const api_1 = require("../utils/api");
 const decoder_1 = require("../utils/decoder");
 const error_1 = require("../utils/error");
+/*
+Fetch Request (Version: 4) => replica_id max_wait_ms min_bytes max_bytes isolation_level [topics]
+  replica_id => INT32
+  max_wait_ms => INT32
+  min_bytes => INT32
+  max_bytes => INT32
+  isolation_level => INT8
+  topics => topic [partitions]
+    topic => STRING
+    partitions => partition fetch_offset partition_max_bytes
+      partition => INT32
+      fetch_offset => INT64
+      partition_max_bytes => INT32
+
+Fetch Response (Version: 4) => throttle_time_ms [responses]
+  throttle_time_ms => INT32
+  responses => topic [partitions]
+    topic => STRING
+    partitions => partition_index error_code high_watermark last_stable_offset [aborted_transactions] records
+      partition_index => INT32
+      error_code => INT16
+      high_watermark => INT64
+      last_stable_offset => INT64
+      aborted_transactions => producer_id first_offset
+        producer_id => INT64
+        first_offset => INT64
+      records => RECORDS
+*/
+const FETCH_V4 = (0, api_1.createApi)({
+    apiKey: 1,
+    apiVersion: 4,
+    requestHeaderVersion: 1,
+    responseHeaderVersion: 0,
+    request: (encoder, data) => encoder
+        .writeInt32(-1) // replica_id
+        .writeInt32(data.maxWaitMs)
+        .writeInt32(data.minBytes)
+        .writeInt32(data.maxBytes)
+        .writeInt8(data.isolationLevel)
+        .writeArray(data.topics, (encoder, topic) => encoder
+        .writeString(topic.topicName)
+        .writeArray(topic.partitions, (encoder, partition) => encoder
+        .writeInt32(partition.partition)
+        .writeInt64(partition.fetchOffset)
+        .writeInt32(partition.partitionMaxBytes))),
+    response: (decoder) => {
+        const result = {
+            throttleTimeMs: decoder.readInt32(),
+            errorCode: 0,
+            sessionId: 0,
+            responses: decoder.readArray((response) => ({
+                topicName: response.readString(),
+                partitions: response.readArray((partition) => ({
+                    partitionIndex: partition.readInt32(),
+                    errorCode: partition.readInt16(),
+                    highWatermark: partition.readInt64(),
+                    lastStableOffset: partition.readInt64(),
+                    logStartOffset: BigInt(0), // Not present in v4 response
+                    abortedTransactions: partition.readArray((abortedTransaction) => ({
+                        producerId: abortedTransaction.readInt64(),
+                        firstOffset: abortedTransaction.readInt64(),
+                    })),
+                    preferredReadReplica: -1, // Not present in v4 response
+                    records: decodeRecordBatch(partition, partition.readInt32()),
+                })),
+            })),
+        };
+        result.responses.forEach((response) => {
+            response.partitions.forEach((partition) => {
+                if (partition.errorCode)
+                    throw new error_1.KafkaTSApiError(partition.errorCode, null, result);
+            });
+        });
+        return result;
+    },
+});
+/*
+Fetch Request (Version: 15) => max_wait_ms min_bytes max_bytes isolation_level session_id session_epoch [topics] [forgotten_topics_data] rack_id _tagged_fields
+  max_wait_ms => INT32
+  min_bytes => INT32
+  max_bytes => INT32
+  isolation_level => INT8
+  session_id => INT32
+  session_epoch => INT32
+  topics => topic_id [partitions] _tagged_fields
+    topic_id => UUID
+    partitions => partition current_leader_epoch fetch_offset last_fetched_epoch log_start_offset partition_max_bytes _tagged_fields
+      partition => INT32
+      current_leader_epoch => INT32
+      fetch_offset => INT64
+      last_fetched_epoch => INT32
+      log_start_offset => INT64
+      partition_max_bytes => INT32
+  forgotten_topics_data => topic_id [partitions] _tagged_fields
+    topic_id => UUID
+    partitions => INT32
+  rack_id => COMPACT_STRING
+
+Fetch Response (Version: 15) => throttle_time_ms error_code session_id [responses] _tagged_fields
+  throttle_time_ms => INT32
+  error_code => INT16
+  session_id => INT32
+  responses => topic_id [partitions] _tagged_fields
+    topic_id => UUID
+    partitions => partition_index error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] preferred_read_replica records _tagged_fields
+      partition_index => INT32
+      error_code => INT16
+      high_watermark => INT64
+      last_stable_offset => INT64
+      log_start_offset => INT64
+      aborted_transactions => producer_id first_offset _tagged_fields
+        producer_id => INT64
+        first_offset => INT64
+      preferred_read_replica => INT32
+      records => COMPACT_RECORDS
+*/
 exports.FETCH = (0, api_1.createApi)({
     apiKey: 1,
     apiVersion: 15,
+    requestHeaderVersion: 2,
+    responseHeaderVersion: 1,
+    fallback: FETCH_V4,
     request: (encoder, data) => encoder
-        .writeUVarInt(0)
         .writeInt32(data.maxWaitMs)
         .writeInt32(data.minBytes)
         .writeInt32(data.maxBytes)
@@ -24,17 +142,16 @@ exports.FETCH = (0, api_1.createApi)({
         .writeInt32(partition.lastFetchedEpoch)
         .writeInt64(partition.logStartOffset)
         .writeInt32(partition.partitionMaxBytes)
-        .writeUVarInt(0))
-        .writeUVarInt(0))
+        .writeTagBuffer())
+        .writeTagBuffer())
         .writeCompactArray(data.forgottenTopicsData, (encoder, forgottenTopic) => encoder
         .writeUUID(forgottenTopic.topicId)
         .writeCompactArray(forgottenTopic.partitions, (encoder, partition) => encoder.writeInt32(partition))
-        .writeUVarInt(0))
+        .writeTagBuffer())
         .writeCompactString(data.rackId)
-        .writeUVarInt(0),
+        .writeTagBuffer(),
     response: async (decoder) => {
         const result = {
-            _tag: decoder.readTagBuffer(),
             throttleTimeMs: decoder.readInt32(),
             errorCode: decoder.readInt16(),
             sessionId: decoder.readInt32(),
@@ -49,15 +166,15 @@ exports.FETCH = (0, api_1.createApi)({
                     abortedTransactions: partition.readCompactArray((abortedTransaction) => ({
                         producerId: abortedTransaction.readInt64(),
                         firstOffset: abortedTransaction.readInt64(),
-                        _tag: abortedTransaction.readTagBuffer(),
+                        tags: abortedTransaction.readTagBuffer(),
                     })),
                     preferredReadReplica: partition.readInt32(),
-                    records: decodeRecordBatch(partition),
-                    _tag: partition.readTagBuffer(),
+                    records: decodeRecordBatch(partition, partition.readUVarInt() - 1),
+                    tags: partition.readTagBuffer(),
                 })),
-                _tag: response.readTagBuffer(),
+                tags: response.readTagBuffer(),
             })),
-            _tag2: decoder.readTagBuffer(),
+            tags: decoder.readTagBuffer(),
         };
         if (result.errorCode)
             throw new error_1.KafkaTSApiError(result.errorCode, null, result);
@@ -70,8 +187,7 @@ exports.FETCH = (0, api_1.createApi)({
         return result;
     },
 });
-const decodeRecordBatch = (decoder) => {
-    const size = decoder.readUVarInt() - 1;
+const decodeRecordBatch = (decoder, size) => {
     if (size <= 0) {
         return [];
     }
@@ -107,6 +223,7 @@ const decodeRecordBatch = (decoder) => {
         const lastOffsetDelta = batchDecoder.readInt32();
         const baseTimestamp = batchDecoder.readInt64();
         const maxTimestamp = batchDecoder.readInt64();
+        const deleteHorizonMs = hasDeleteHorizonMs ? baseTimestamp : null;
         const producerId = batchDecoder.readInt64();
         const producerEpoch = batchDecoder.readInt16();
         const baseSequence = batchDecoder.readInt32();
@@ -123,6 +240,7 @@ const decodeRecordBatch = (decoder) => {
             isTransactional,
             isControlBatch,
             hasDeleteHorizonMs,
+            deleteHorizonMs,
             lastOffsetDelta,
             baseTimestamp,
             maxTimestamp,

package/dist/api/find-coordinator.d.ts

@@ -2,20 +2,22 @@ export declare const KEY_TYPE: {
     GROUP: number;
     TRANSACTION: number;
 };
-export declare const FIND_COORDINATOR: import("../utils/api").Api<{
+type FindCoordinatorRequest = {
     keyType: number;
     keys: string[];
-}, {
-    _tag: void;
+};
+type FindCoordinatorResponse = {
     throttleTimeMs: number;
     coordinators: {
-        key: string | null;
+        key: string;
         nodeId: number;
         host: string;
         port: number;
         errorCode: number;
         errorMessage: string | null;
-        _tag: void;
+        tags: Record<number, Buffer>;
     }[];
-    _tag2: void;
-}>;
+    tags: Record<number, Buffer>;
+};
+export declare const FIND_COORDINATOR: import("../utils/api").Api<FindCoordinatorRequest, FindCoordinatorResponse>;
+export {};

package/dist/api/find-coordinator.js

@@ -7,17 +7,75 @@ exports.KEY_TYPE = {
     GROUP: 0,
     TRANSACTION: 1,
 };
+/*
+FindCoordinator Request (Version: 1) => key key_type
+  key => STRING
+  key_type => INT8
+
+FindCoordinator Response (Version: 1) => throttle_time_ms error_code error_message node_id host port
+  throttle_time_ms => INT32
+  error_code => INT16
+  error_message => NULLABLE_STRING
+  node_id => INT32
+  host => STRING
+  port => INT32
+*/
+const FIND_COORDINATOR_V1 = (0, api_1.createApi)({
+    apiKey: 10,
+    apiVersion: 1,
+    requestHeaderVersion: 1,
+    responseHeaderVersion: 0,
+    request: (encoder, data) => encoder.writeString(data.keys[0]).writeInt8(data.keyType),
+    response: (decoder) => {
+        const result = {
+            throttleTimeMs: decoder.readInt32(),
+            errorCode: decoder.readInt16(),
+            errorMessage: decoder.readString(),
+            coordinators: [
+                {
+                    key: '', // Key not present in v1 response
+                    nodeId: decoder.readInt32(),
+                    host: decoder.readString(),
+                    port: decoder.readInt32(),
+                    errorCode: 0,
+                    errorMessage: null,
+                    tags: {},
+                },
+            ],
+            tags: {},
+        };
+        if (result.errorCode)
+            throw new error_1.KafkaTSApiError(result.errorCode, result.errorMessage, result);
+        return result;
+    },
+});
+/*
+FindCoordinator Request (Version: 4) => key_type [coordinator_keys] _tagged_fields
+  key_type => INT8
+  coordinator_keys => COMPACT_STRING
+
+FindCoordinator Response (Version: 4) => throttle_time_ms [coordinators] _tagged_fields
+  throttle_time_ms => INT32
+  coordinators => key node_id host port error_code error_message _tagged_fields
+    key => COMPACT_STRING
+    node_id => INT32
+    host => COMPACT_STRING
+    port => INT32
+    error_code => INT16
+    error_message => COMPACT_NULLABLE_STRING
+*/
 exports.FIND_COORDINATOR = (0, api_1.createApi)({
     apiKey: 10,
     apiVersion: 4,
+    fallback: FIND_COORDINATOR_V1,
+    requestHeaderVersion: 2,
+    responseHeaderVersion: 1,
     request: (encoder, data) => encoder
-        .writeUVarInt(0)
         .writeInt8(data.keyType)
         .writeCompactArray(data.keys, (encoder, key) => encoder.writeCompactString(key))
-        .writeUVarInt(0),
+        .writeTagBuffer(),
     response: (decoder) => {
         const result = {
-            _tag: decoder.readTagBuffer(),
             throttleTimeMs: decoder.readInt32(),
             coordinators: decoder.readCompactArray((decoder) => ({
                 key: decoder.readCompactString(),
@@ -26,9 +84,9 @@ exports.FIND_COORDINATOR = (0, api_1.createApi)({
                 port: decoder.readInt32(),
                 errorCode: decoder.readInt16(),
                 errorMessage: decoder.readCompactString(),
-                _tag: decoder.readTagBuffer(),
+                tags: decoder.readTagBuffer(),
             })),
-            _tag2: decoder.readTagBuffer(),
+            tags: decoder.readTagBuffer(),
         };
         result.coordinators.forEach((coordinator) => {
             if (coordinator.errorCode)

package/dist/api/heartbeat.d.ts

@@ -1,11 +1,13 @@
-export declare const HEARTBEAT: import("../utils/api").Api<{
+type HeartbeatRequest = {
     groupId: string;
     generationId: number;
     memberId: string;
     groupInstanceId: string | null;
-}, {
-    _tag: void;
+};
+type HeartbeatResponse = {
     throttleTimeMs: number;
     errorCode: number;
-    _tag2: void;
-}>;
+    tags: Record<number, Buffer>;
+};
+export declare const HEARTBEAT: import("../utils/api").Api<HeartbeatRequest, HeartbeatResponse>;
+export {};

package/dist/api/heartbeat.js

@@ -3,22 +3,60 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.HEARTBEAT = void 0;
 const api_1 = require("../utils/api");
 const error_1 = require("../utils/error");
+/*
+Heartbeat Request (Version: 0) => group_id generation_id member_id
+  group_id => STRING
+  generation_id => INT32
+  member_id => STRING
+
+Heartbeat Response (Version: 0) => error_code
+  error_code => INT16
+*/
+const HEARTBEAT_V0 = (0, api_1.createApi)({
+    apiKey: 12,
+    apiVersion: 0,
+    requestHeaderVersion: 1,
+    responseHeaderVersion: 0,
+    request: (encoder, data) => encoder.writeString(data.groupId).writeInt32(data.generationId).writeString(data.memberId),
+    response: (decoder) => {
+        const result = {
+            throttleTimeMs: 0,
+            errorCode: decoder.readInt16(),
+            tags: {},
+        };
+        if (result.errorCode)
+            throw new error_1.KafkaTSApiError(result.errorCode, null, result);
+        return result;
+    },
+});
+/*
+Heartbeat Request (Version: 4) => group_id generation_id member_id group_instance_id _tagged_fields
+  group_id => COMPACT_STRING
+  generation_id => INT32
+  member_id => COMPACT_STRING
+  group_instance_id => COMPACT_NULLABLE_STRING
+
+Heartbeat Response (Version: 4) => throttle_time_ms error_code _tagged_fields
+  throttle_time_ms => INT32
+  error_code => INT16
+*/
 exports.HEARTBEAT = (0, api_1.createApi)({
     apiKey: 12,
     apiVersion: 4,
+    fallback: HEARTBEAT_V0,
+    requestHeaderVersion: 2,
+    responseHeaderVersion: 1,
     request: (encoder, data) => encoder
-        .writeUVarInt(0)
         .writeCompactString(data.groupId)
         .writeInt32(data.generationId)
         .writeCompactString(data.memberId)
         .writeCompactString(data.groupInstanceId)
-        .writeUVarInt(0),
+        .writeTagBuffer(),
     response: (decoder) => {
         const result = {
-            _tag: decoder.readTagBuffer(),
             throttleTimeMs: decoder.readInt32(),
             errorCode: decoder.readInt16(),
-            _tag2: decoder.readTagBuffer(),
+            tags: decoder.readTagBuffer(),
         };
         if (result.errorCode)
             throw new error_1.KafkaTSApiError(result.errorCode, null, result);