kafka-ts 1.3.1-beta.1 → 1.3.1-beta.3

package/README.md

@@ -127,9 +127,9 @@ The existing high-level libraries (e.g. kafkajs) are missing a few crucial featu
 
 ### New features compared to kafkajs
 
--   **Static consumer membership** - Rebalancing during rolling deployments causes delays. Using `groupInstanceId` in addition to `groupId` can avoid rebalancing and continue consuming partitions in the existing assignment.
--   **Consuming messages without consumer groups** - When you don't need the consumer to track the partition offsets, you can simply create a consumer without groupId and always either start consuming messages from the beginning or from the latest partition offset.
--   **Low-level API requests** - It's possible to communicate directly with the Kafka cluster using the kafka api protocol.
+- **Static consumer membership** - Rebalancing during rolling deployments causes delays. Using `groupInstanceId` in addition to `groupId` can avoid rebalancing and continue consuming partitions in the existing assignment.
+- **Consuming messages without consumer groups** - When you don't need the consumer to track the partition offsets, you can simply create a consumer without groupId and always either start consuming messages from the beginning or from the latest partition offset.
+- **Low-level API requests** - It's possible to communicate directly with the Kafka cluster using the kafka api protocol.
 
 ## Configuration
 
@@ -145,10 +145,10 @@ The existing high-level libraries (e.g. kafkajs) are missing a few crucial featu
 
 #### Supported SASL mechanisms
 
--   PLAIN: `saslPlain({ username, password })`
--   SCRAM-SHA-256: `saslScramSha256({ username, password })`
--   SCRAM-SHA-512: `saslScramSha512({ username, password })`
--   OAUTHBEARER: `oAuthBearer(oAuthAuthenticator({ endpoint, clientId, clientSecret }))`
+- PLAIN: `saslPlain({ username, password })`
+- SCRAM-SHA-256: `saslScramSha256({ username, password })`
+- SCRAM-SHA-512: `saslScramSha512({ username, password })`
+- OAUTHBEARER: `oAuthBearer(oAuthAuthenticator({ endpoint, clientId, clientSecret }))`
 
 Custom SASL mechanisms can be implemented following the `SASLProvider` interface. See [src/auth](./src/auth) for examples.
 

package/dist/api/alter-configs.d.ts

@@ -0,0 +1,26 @@
+export declare const ALTER_CONFIGS: import("../utils/api").Api<{
+    resources: {
+        resourceType: number;
+        resourceName: string;
+        configs: {
+            name: string;
+            value: string | null;
+        }[];
+    }[];
+    validateOnly: boolean;
+}, {
+    _tag: void;
+    responses: {
+        name: string | null;
+        partitionResponses: {
+            index: number;
+            errorCode: number;
+            baseOffset: bigint;
+            logAppendTimeMs: bigint;
+            _tag: void;
+        }[];
+        _tag: void;
+    }[];
+    throttleTimeMs: number;
+    _tag2: void;
+}>;

package/dist/api/alter-configs.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ALTER_CONFIGS = void 0;
+const api_1 = require("../utils/api");
+exports.ALTER_CONFIGS = (0, api_1.createApi)({
+    apiKey: 33,
+    apiVersion: 2,
+    request: (encoder, data) => encoder
+        .writeUVarInt(0)
+        .writeCompactArray(data.resources, (encoder, item) => encoder
+        .writeInt8(item.resourceType)
+        .writeCompactString(item.resourceName)
+        .writeCompactArray(item.configs, (encoder, item) => encoder.writeCompactString(item.name).writeCompactString(item.value).writeUVarInt(0))
+        .writeUVarInt(0))
+        .writeBoolean(data.validateOnly)
+        .writeUVarInt(0),
+    response: (decoder) => ({
+        _tag: decoder.readTagBuffer(),
+        responses: decoder.readCompactArray((decoder) => ({
+            name: decoder.readCompactString(),
+            partitionResponses: decoder.readCompactArray((decoder) => ({
+                index: decoder.readInt32(),
+                errorCode: decoder.readInt16(),
+                baseOffset: decoder.readInt64(),
+                logAppendTimeMs: decoder.readInt64(),
+                _tag: decoder.readTagBuffer(),
+            })),
+            _tag: decoder.readTagBuffer(),
+        })),
+        throttleTimeMs: decoder.readInt32(),
+        _tag2: decoder.readTagBuffer(),
+    }),
+});

package/dist/api/api-versions.d.ts

@@ -1,4 +1,5 @@
-export declare const API_VERSIONS: import("../utils/api.js").Api<unknown, {
+type ApiVersionsRequest = {};
+type ApiVersionsResponse = {
     errorCode: number;
     versions: {
         apiKey: number;
@@ -6,4 +7,6 @@ export declare const API_VERSIONS: import("../utils/api.js").Api<unknown, {
         maxVersion: number;
     }[];
     throttleTimeMs: number;
-}>;
+};
+export declare const API_VERSIONS: import("../utils/api.js").Api<ApiVersionsRequest, ApiVersionsResponse>;
+export {};

package/dist/api/api-versions.js

@@ -3,9 +3,22 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.API_VERSIONS = void 0;
 const api_js_1 = require("../utils/api.js");
 const error_js_1 = require("../utils/error.js");
+/*
+ApiVersions Request (Version: 2) =>
+
+ApiVersions Response (Version: 2) => error_code [api_keys] throttle_time_ms
+  error_code => INT16
+  api_keys => api_key min_version max_version
+    api_key => INT16
+    min_version => INT16
+    max_version => INT16
+  throttle_time_ms => INT32
+*/
 exports.API_VERSIONS = (0, api_js_1.createApi)({
     apiKey: 18,
     apiVersion: 2,
+    requestHeaderVersion: 1,
+    responseHeaderVersion: 0,
     request: (encoder) => encoder,
     response: (decoder) => {
         const result = {

package/dist/api/create-topics.d.ts

@@ -1,4 +1,4 @@
-export declare const CREATE_TOPICS: import("../utils/api").Api<{
+type CreateTopicsRequest = {
     topics: {
         name: string;
         numPartitions?: number;
@@ -14,25 +14,27 @@ export declare const CREATE_TOPICS: import("../utils/api").Api<{
     }[];
     timeoutMs?: number;
     validateOnly?: boolean;
-}, {
-    _tag: void;
+};
+type CreateTopicsResponse = {
     throttleTimeMs: number;
     topics: {
-        name: string | null;
-        topicId: string;
+        name: string;
+        _topicId: string;
         errorCode: number;
         errorMessage: string | null;
-        numPartitions: number;
-        replicationFactor: number;
+        _numPartitions: number;
+        _replicationFactor: number;
         configs: {
-            name: string | null;
+            name: string;
             value: string | null;
             readOnly: boolean;
             configSource: number;
             isSensitive: boolean;
-            _tag: void;
+            tags: Record<number, Buffer>;
         }[];
-        _tag: void;
+        tags: Record<number, Buffer>;
     }[];
-    _tag2: void;
-}>;
+    tags: Record<number, Buffer>;
+};
+export declare const CREATE_TOPICS: import("../utils/api").Api<CreateTopicsRequest, CreateTopicsResponse>;
+export {};

package/dist/api/create-topics.js

@@ -3,11 +3,104 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.CREATE_TOPICS = void 0;
 const api_1 = require("../utils/api");
 const error_1 = require("../utils/error");
+/*
+CreateTopics Request (Version: 2) => [topics] timeout_ms validate_only
+  topics => name num_partitions replication_factor [assignments] [configs]
+    name => STRING
+    num_partitions => INT32
+    replication_factor => INT16
+    assignments => partition_index [broker_ids]
+      partition_index => INT32
+      broker_ids => INT32
+    configs => name value
+      name => STRING
+      value => NULLABLE_STRING
+  timeout_ms => INT32
+  validate_only => BOOLEAN
+
+CreateTopics Response (Version: 2) => throttle_time_ms [topics]
+  throttle_time_ms => INT32
+  topics => name error_code error_message
+    name => STRING
+    error_code => INT16
+    error_message => NULLABLE_STRING
+*/
+const CREATE_TOPICS_V2 = (0, api_1.createApi)({
+    apiKey: 19,
+    apiVersion: 2,
+    requestHeaderVersion: 1,
+    responseHeaderVersion: 0,
+    request: (encoder, data) => encoder
+        .writeArray(data.topics, (encoder, topic) => encoder
+        .writeString(topic.name)
+        .writeInt32(topic.numPartitions ?? -1)
+        .writeInt16(topic.replicationFactor ?? -1)
+        .writeArray(topic.assignments ?? [], (encoder, assignment) => encoder
+        .writeInt32(assignment.partitionIndex)
+        .writeArray(assignment.brokerIds, (encoder, brokerId) => encoder.writeInt32(brokerId)))
+        .writeArray(topic.configs ?? [], (encoder, config) => encoder.writeString(config.name).writeString(config.value)))
+        .writeInt32(data.timeoutMs ?? 10_000)
+        .writeBoolean(data.validateOnly ?? false),
+    response: (decoder) => {
+        const result = {
+            throttleTimeMs: decoder.readInt32(),
+            topics: decoder.readArray((topic) => ({
+                name: topic.readString(),
+                _topicId: '', // TopicId not present in v2 response
+                errorCode: topic.readInt16(),
+                errorMessage: topic.readString(),
+                _numPartitions: 0, // Not present in v2 response
+                _replicationFactor: 0, // Not present in v2 response
+                configs: [], // Not present in v2 response
+                tags: {},
+            })),
+            tags: {},
+        };
+        result.topics.forEach((topic) => {
+            if (topic.errorCode)
+                throw new error_1.KafkaTSApiError(topic.errorCode, topic.errorMessage, result);
+        });
+        return result;
+    },
+});
+/*
+CreateTopics Request (Version: 7) => [topics] timeout_ms validate_only _tagged_fields
+  topics => name num_partitions replication_factor [assignments] [configs] _tagged_fields
+    name => COMPACT_STRING
+    num_partitions => INT32
+    replication_factor => INT16
+    assignments => partition_index [broker_ids] _tagged_fields
+      partition_index => INT32
+      broker_ids => INT32
+    configs => name value _tagged_fields
+      name => COMPACT_STRING
+      value => COMPACT_NULLABLE_STRING
+  timeout_ms => INT32
+  validate_only => BOOLEAN
+
+CreateTopics Response (Version: 7) => throttle_time_ms [topics] _tagged_fields
+  throttle_time_ms => INT32
+  topics => name topic_id error_code error_message num_partitions replication_factor [configs] _tagged_fields
+    name => COMPACT_STRING
+    topic_id => UUID
+    error_code => INT16
+    error_message => COMPACT_NULLABLE_STRING
+    num_partitions => INT32
+    replication_factor => INT16
+    configs => name value read_only config_source is_sensitive _tagged_fields
+      name => COMPACT_STRING
+      value => COMPACT_NULLABLE_STRING
+      read_only => BOOLEAN
+      config_source => INT8
+      is_sensitive => BOOLEAN
+*/
 exports.CREATE_TOPICS = (0, api_1.createApi)({
     apiKey: 19,
     apiVersion: 7,
+    fallback: CREATE_TOPICS_V2,
+    requestHeaderVersion: 2,
+    responseHeaderVersion: 1,
     request: (encoder, data) => encoder
-        .writeUVarInt(0)
         .writeCompactArray(data.topics, (encoder, topic) => encoder
         .writeCompactString(topic.name)
         .writeInt32(topic.numPartitions ?? -1)
@@ -15,34 +108,33 @@ exports.CREATE_TOPICS = (0, api_1.createApi)({
         .writeCompactArray(topic.assignments ?? [], (encoder, assignment) => encoder
         .writeInt32(assignment.partitionIndex)
         .writeCompactArray(assignment.brokerIds, (encoder, brokerId) => encoder.writeInt32(brokerId))
-        .writeUVarInt(0))
-        .writeCompactArray(topic.configs ?? [], (encoder, config) => encoder.writeCompactString(config.name).writeCompactString(config.value).writeUVarInt(0))
-        .writeUVarInt(0))
+        .writeTagBuffer())
+        .writeCompactArray(topic.configs ?? [], (encoder, config) => encoder.writeCompactString(config.name).writeCompactString(config.value).writeTagBuffer())
+        .writeTagBuffer())
         .writeInt32(data.timeoutMs ?? 10_000)
         .writeBoolean(data.validateOnly ?? false)
-        .writeUVarInt(0),
+        .writeTagBuffer(),
     response: (decoder) => {
         const result = {
-            _tag: decoder.readTagBuffer(),
             throttleTimeMs: decoder.readInt32(),
             topics: decoder.readCompactArray((topic) => ({
                 name: topic.readCompactString(),
-                topicId: topic.readUUID(),
+                _topicId: topic.readUUID(),
                 errorCode: topic.readInt16(),
                 errorMessage: topic.readCompactString(),
-                numPartitions: topic.readInt32(),
-                replicationFactor: topic.readInt16(),
+                _numPartitions: topic.readInt32(),
+                _replicationFactor: topic.readInt16(),
                 configs: topic.readCompactArray((config) => ({
                     name: config.readCompactString(),
                     value: config.readCompactString(),
                     readOnly: config.readBoolean(),
                     configSource: config.readInt8(),
                     isSensitive: config.readBoolean(),
-                    _tag: config.readTagBuffer(),
+                    tags: config.readTagBuffer(),
                 })),
-                _tag: topic.readTagBuffer(),
+                tags: topic.readTagBuffer(),
             })),
-            _tag2: decoder.readTagBuffer(),
+            tags: decoder.readTagBuffer(),
         };
         result.topics.forEach((topic) => {
             if (topic.errorCode)

package/dist/api/delete-topics.d.ts

@@ -1,18 +1,20 @@
-export declare const DELETE_TOPICS: import("../utils/api").Api<{
+type DeleteTopicsRequest = {
     topics: {
-        name: string | null;
+        name: string;
         topicId: string | null;
     }[];
     timeoutMs?: number;
-}, {
-    _tag: void;
+};
+type DeleteTopicsResponse = {
     throttleTimeMs: number;
     responses: {
         name: string | null;
-        topicId: string;
+        _topicId: string;
         errorCode: number;
         errorMessage: string | null;
-        _tag: void;
+        tags: Record<number, Buffer>;
     }[];
-    _tag2: void;
-}>;
+    tags: Record<number, Buffer>;
+};
+export declare const DELETE_TOPICS: import("../utils/api").Api<DeleteTopicsRequest, DeleteTopicsResponse>;
+export {};

package/dist/api/delete-topics.js

@@ -3,26 +3,80 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DELETE_TOPICS = void 0;
 const api_1 = require("../utils/api");
 const error_1 = require("../utils/error");
+/*
+DeleteTopics Request (Version: 1) => [topic_names] timeout_ms
+  topic_names => STRING
+  timeout_ms => INT32
+
+DeleteTopics Response (Version: 1) => throttle_time_ms [responses]
+  throttle_time_ms => INT32
+  responses => name error_code
+    name => STRING
+    error_code => INT16
+*/
+const DELETE_TOPICS_V1 = (0, api_1.createApi)({
+    apiKey: 20,
+    apiVersion: 1,
+    requestHeaderVersion: 1,
+    responseHeaderVersion: 0,
+    request: (encoder, data) => encoder
+        .writeArray(data.topics, (encoder, topic) => encoder.writeString(topic.name))
+        .writeInt32(data.timeoutMs ?? 10_000),
+    response: (decoder) => {
+        const result = {
+            throttleTimeMs: decoder.readInt32(),
+            responses: decoder.readArray((decoder) => ({
+                name: decoder.readString(),
+                _topicId: '', // TopicId not present in v1 response
+                errorCode: decoder.readInt16(),
+                errorMessage: null,
+                tags: {},
+            })),
+            tags: {},
+        };
+        result.responses.forEach((response) => {
+            if (response.errorCode)
+                throw new error_1.KafkaTSApiError(response.errorCode, response.errorMessage, result);
+        });
+        return result;
+    },
+});
+/*
+DeleteTopics Request (Version: 6) => [topics] timeout_ms _tagged_fields
+  topics => name topic_id _tagged_fields
+    name => COMPACT_NULLABLE_STRING
+    topic_id => UUID
+  timeout_ms => INT32
+
+DeleteTopics Response (Version: 6) => throttle_time_ms [responses] _tagged_fields
+  throttle_time_ms => INT32
+  responses => name topic_id error_code error_message _tagged_fields
+    name => COMPACT_NULLABLE_STRING
+    topic_id => UUID
+    error_code => INT16
+    error_message => COMPACT_NULLABLE_STRING
+*/
 exports.DELETE_TOPICS = (0, api_1.createApi)({
     apiKey: 20,
     apiVersion: 6,
+    fallback: DELETE_TOPICS_V1,
+    requestHeaderVersion: 2,
+    responseHeaderVersion: 1,
     request: (encoder, data) => encoder
-        .writeUVarInt(0)
-        .writeCompactArray(data.topics, (encoder, topic) => encoder.writeCompactString(topic.name).writeUUID(topic.topicId).writeUVarInt(0))
+        .writeCompactArray(data.topics, (encoder, topic) => encoder.writeCompactString(topic.name).writeUUID(topic.topicId).writeTagBuffer())
         .writeInt32(data.timeoutMs ?? 10_000)
-        .writeUVarInt(0),
+        .writeTagBuffer(),
     response: (decoder) => {
         const result = {
-            _tag: decoder.readTagBuffer(),
             throttleTimeMs: decoder.readInt32(),
             responses: decoder.readCompactArray((decoder) => ({
                 name: decoder.readCompactString(),
-                topicId: decoder.readUUID(),
+                _topicId: decoder.readUUID(),
                 errorCode: decoder.readInt16(),
                 errorMessage: decoder.readCompactString(),
-                _tag: decoder.readTagBuffer(),
+                tags: decoder.readTagBuffer(),
             })),
-            _tag2: decoder.readTagBuffer(),
+            tags: decoder.readTagBuffer(),
         };
         result.responses.forEach((response) => {
             if (response.errorCode)

package/dist/api/fetch.d.ts

@@ -2,8 +2,7 @@ export declare const enum IsolationLevel {
     READ_UNCOMMITTED = 0,
     READ_COMMITTED = 1
 }
-export type FetchResponse = Awaited<ReturnType<(typeof FETCH)['response']>>;
-export declare const FETCH: import("../utils/api").Api<{
+type FetchRequest = {
     maxWaitMs: number;
     minBytes: number;
     maxBytes: number;
@@ -12,6 +11,7 @@ export declare const FETCH: import("../utils/api").Api<{
     sessionEpoch: number;
     topics: {
         topicId: string;
+        topicName: string;
         partitions: {
             partition: number;
             currentLeaderEpoch: number;
@@ -23,16 +23,20 @@ export declare const FETCH: import("../utils/api").Api<{
     }[];
     forgottenTopicsData: {
         topicId: string;
+        topicName: string;
         partitions: number[];
     }[];
     rackId: string;
-}, {
-    _tag: void;
+};
+export type FetchResponse = {
     throttleTimeMs: number;
     errorCode: number;
     sessionId: number;
-    responses: {
+    responses: (({
         topicId: string;
+    } | {
+        topicName: string;
+    }) & {
         partitions: {
             partitionIndex: number;
             errorCode: number;
@@ -42,7 +46,6 @@ export declare const FETCH: import("../utils/api").Api<{
             abortedTransactions: {
                 producerId: bigint;
                 firstOffset: bigint;
-                _tag: void;
             }[];
             preferredReadReplica: number;
             records: {
@@ -53,10 +56,11 @@ export declare const FETCH: import("../utils/api").Api<{
                 crc: number;
                 attributes: number;
                 compression: number;
-                timestampType: string;
+                timestampType: 'CreateTime' | 'LogAppendTime';
                 isTransactional: boolean;
                 isControlBatch: boolean;
                 hasDeleteHorizonMs: boolean;
+                deleteHorizonMs: bigint | null;
                 lastOffsetDelta: number;
                 baseTimestamp: bigint;
                 maxTimestamp: bigint;
@@ -75,9 +79,8 @@ export declare const FETCH: import("../utils/api").Api<{
                     }[];
                 }[];
             }[];
-            _tag: void;
         }[];
-        _tag: void;
-    }[];
-    _tag2: void;
-}>;
+    })[];
+};
+export declare const FETCH: import("../utils/api").Api<FetchRequest, FetchResponse>;
+export {};