kafka-ts 0.0.1-beta.4 → 0.0.1-beta.6

package/.github/workflows/release.yml

@@ -1,17 +1,30 @@
 name: Publish package
 on:
-    release:
-        types: [published]
+    push:
+        tags:
+            - 'v*'
 jobs:
     build:
         runs-on: ubuntu-latest
+        permissions:
+          contents: write
         steps:
-            - uses: actions/checkout@v4
-            - uses: actions/setup-node@v4
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Setup node
+              uses: actions/setup-node@v4
               with:
                   node-version: '20.x'
                   registry-url: 'https://registry.npmjs.org'
-            - run: npm ci
-            - run: npm publish
+            - name: Install dependencies
+              run: npm ci
+            - name: Build
+              run: npm run build
+            - name: Create release
+              env:
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: gh release create ${{ github.ref_name }} --generate-notes
+            - name: Publish npm package
+              run: npm publish
               env:
                   NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}

package/README.md

@@ -2,7 +2,7 @@
 
 **KafkaTS** is a Apache Kafka client library for Node.js. It provides both a low-level API for communicating directly with the Apache Kafka cluster and high-level APIs for publishing and subscribing to Kafka topics.
 
-**Please note that this project is still in early development and is not yet ready for production use. The interface before stable release is subject to change.**
+Supported Kafka versions: 3.7.0 and later
 
 ## Installation
 
@@ -96,12 +96,20 @@ The existing high-level libraries (e.g. kafkajs) are missing a few crucial featu
 
 ### `createKafkaClient()`
 
-| Name             | Type                   | Required | Default | Description                                           |
-| ---------------- | ---------------------- | -------- | ------- | ----------------------------------------------------- |
-| clientId         | string                 | false    | _null_  | The client id used for all requests.                  |
-| bootstrapServers | TcpSocketConnectOpts[] | true     |         | List of kafka brokers for initial cluster discovery.  |
-| sasl             | SASLProvider           | false    |         | SASL provider (see "Supported SASL mechanisms" below) |
-| ssl              | TLSSocketOptions       | false    |         | SSL configuration.                                    |
+| Name             | Type                   | Required | Default | Description                                          |
+| ---------------- | ---------------------- | -------- | ------- | ---------------------------------------------------- |
+| clientId         | string                 | false    | _null_  | The client id used for all requests.                 |
+| bootstrapServers | TcpSocketConnectOpts[] | true     |         | List of kafka brokers for initial cluster discovery. |
+| sasl             | SASLProvider           | false    |         | SASL provider                                        |
+| ssl              | TLSSocketOptions       | false    |         | SSL configuration.                                   |
+
+#### Supported SASL mechanisms
+
+-   PLAIN: `saslPlain({ username, password })`
+-   SCRAM-SHA-256: `saslScramSha256({ username, password })`
+-   SCRAM-SHA-512: `saslScramSha512({ username, password })`
+
+Custom SASL mechanisms can be implemented following the `SASLProvider` interface. See [src/auth](./src/auth) for examples.
 
 ### `kafka.startConsumer()`
 
@@ -151,17 +159,3 @@ The existing high-level libraries (e.g. kafkajs) are missing a few crucial featu
 | key       | Buffer \| null         | false    | _null_  | Message key                                                                                                                |
 | value     | Buffer \| null         | true     |         | Message value                                                                                                              |
 | headers   | Record<string, string> | false    | _null_  | Message headers                                                                                                            |
-
-### Supported SASL mechanisms
-
--   PLAIN: `saslPlain({ username, password })`
--   SCRAM-SHA-256: `saslScramSha256({ username, password })`
--   SCRAM-SHA-512: `saslScramSha512({ username, password })`
-
-Custom SASL mechanisms can be implemented following the `SASLProvider` interface. See [src/auth](./src/auth) for examples.
-
-## Backlog
-
-Minimal set of features left to implement before a stable release:
-
--   API versioning (Currently only tested against Kafka 3.7+)

package/dist/api/api-versions.d.ts

@@ -0,0 +1,9 @@
+export declare const API_VERSIONS: import("../utils/api.js").Api<unknown, {
+    errorCode: number;
+    versions: {
+        apiKey: number;
+        minVersion: number;
+        maxVersion: number;
+    }[];
+    throttleTimeMs: number;
+}>;

package/dist/api/api-versions.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.API_VERSIONS = void 0;
+const api_js_1 = require("../utils/api.js");
+const error_js_1 = require("../utils/error.js");
+exports.API_VERSIONS = (0, api_js_1.createApi)({
+    apiKey: 18,
+    apiVersion: 2,
+    request: (encoder) => encoder,
+    response: (decoder) => {
+        const result = {
+            errorCode: decoder.readInt16(),
+            versions: decoder.readArray((version) => ({
+                apiKey: version.readInt16(),
+                minVersion: version.readInt16(),
+                maxVersion: version.readInt16(),
+            })),
+            throttleTimeMs: decoder.readInt32(),
+        };
+        if (result.errorCode)
+            throw new error_js_1.KafkaTSApiError(result.errorCode, null, result);
+        return result;
+    },
+});

package/dist/api/create-topics.d.ts

@@ -0,0 +1,38 @@
+export declare const CREATE_TOPICS: import("../utils/api").Api<{
+    topics: {
+        name: string;
+        numPartitions: number;
+        replicationFactor: number;
+        assignments: {
+            partitionIndex: number;
+            brokerIds: number[];
+        }[];
+        configs: {
+            name: string;
+            value: string | null;
+        }[];
+    }[];
+    timeoutMs: number;
+    validateOnly: boolean;
+}, {
+    _tag: void;
+    throttleTimeMs: number;
+    topics: {
+        name: string | null;
+        topicId: string;
+        errorCode: number;
+        errorMessage: string | null;
+        numPartitions: number;
+        replicationFactor: number;
+        configs: {
+            name: string | null;
+            value: string | null;
+            readOnly: boolean;
+            configSource: number;
+            isSensitive: boolean;
+            _tag: void;
+        }[];
+        _tag: void;
+    }[];
+    _tag2: void;
+}>;

package/dist/api/create-topics.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CREATE_TOPICS = void 0;
+const api_1 = require("../utils/api");
+const error_1 = require("../utils/error");
+exports.CREATE_TOPICS = (0, api_1.createApi)({
+    apiKey: 19,
+    apiVersion: 7,
+    request: (encoder, data) => encoder
+        .writeUVarInt(0)
+        .writeCompactArray(data.topics, (encoder, topic) => encoder
+        .writeCompactString(topic.name)
+        .writeInt32(topic.numPartitions)
+        .writeInt16(topic.replicationFactor)
+        .writeCompactArray(topic.assignments, (encoder, assignment) => encoder
+        .writeInt32(assignment.partitionIndex)
+        .writeCompactArray(assignment.brokerIds, (encoder, brokerId) => encoder.writeInt32(brokerId))
+        .writeUVarInt(0))
+        .writeCompactArray(topic.configs, (encoder, config) => encoder.writeCompactString(config.name).writeCompactString(config.value).writeUVarInt(0))
+        .writeUVarInt(0))
+        .writeInt32(data.timeoutMs)
+        .writeBoolean(data.validateOnly)
+        .writeUVarInt(0),
+    response: (decoder) => {
+        const result = {
+            _tag: decoder.readTagBuffer(),
+            throttleTimeMs: decoder.readInt32(),
+            topics: decoder.readCompactArray((topic) => ({
+                name: topic.readCompactString(),
+                topicId: topic.readUUID(),
+                errorCode: topic.readInt16(),
+                errorMessage: topic.readCompactString(),
+                numPartitions: topic.readInt32(),
+                replicationFactor: topic.readInt16(),
+                configs: topic.readCompactArray((config) => ({
+                    name: config.readCompactString(),
+                    value: config.readCompactString(),
+                    readOnly: config.readBoolean(),
+                    configSource: config.readInt8(),
+                    isSensitive: config.readBoolean(),
+                    _tag: config.readTagBuffer(),
+                })),
+                _tag: topic.readTagBuffer(),
+            })),
+            _tag2: decoder.readTagBuffer(),
+        };
+        result.topics.forEach((topic) => {
+            if (topic.errorCode)
+                throw new error_1.KafkaTSApiError(topic.errorCode, topic.errorMessage, result);
+        });
+        return result;
+    },
+});

package/dist/api/delete-topics.d.ts

@@ -0,0 +1,18 @@
+export declare const DELETE_TOPICS: import("../utils/api").Api<{
+    topics: {
+        name: string | null;
+        topicId: string | null;
+    }[];
+    timeoutMs: number;
+}, {
+    _tag: void;
+    throttleTimeMs: number;
+    responses: {
+        name: string | null;
+        topicId: string;
+        errorCode: number;
+        errorMessage: string | null;
+        _tag: void;
+    }[];
+    _tag2: void;
+}>;

package/dist/api/delete-topics.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DELETE_TOPICS = void 0;
+const api_1 = require("../utils/api");
+const error_1 = require("../utils/error");
+exports.DELETE_TOPICS = (0, api_1.createApi)({
+    apiKey: 20,
+    apiVersion: 6,
+    request: (encoder, data) => encoder
+        .writeUVarInt(0)
+        .writeCompactArray(data.topics, (encoder, topic) => encoder.writeCompactString(topic.name).writeUUID(topic.topicId).writeUVarInt(0))
+        .writeInt32(data.timeoutMs)
+        .writeUVarInt(0),
+    response: (decoder) => {
+        const result = {
+            _tag: decoder.readTagBuffer(),
+            throttleTimeMs: decoder.readInt32(),
+            responses: decoder.readCompactArray((decoder) => ({
+                name: decoder.readCompactString(),
+                topicId: decoder.readUUID(),
+                errorCode: decoder.readInt16(),
+                errorMessage: decoder.readCompactString(),
+                _tag: decoder.readTagBuffer(),
+            })),
+            _tag2: decoder.readTagBuffer(),
+        };
+        result.responses.forEach((response) => {
+            if (response.errorCode)
+                throw new error_1.KafkaTSApiError(response.errorCode, response.errorMessage, result);
+        });
+        return result;
+    },
+});

package/dist/api/fetch.d.ts

@@ -0,0 +1,84 @@
+/// <reference types="node" />
+export declare const enum IsolationLevel {
+    READ_UNCOMMITTED = 0,
+    READ_COMMITTED = 1
+}
+export type FetchResponse = Awaited<ReturnType<(typeof FETCH)['response']>>;
+export declare const FETCH: import("../utils/api").Api<{
+    maxWaitMs: number;
+    minBytes: number;
+    maxBytes: number;
+    isolationLevel: IsolationLevel;
+    sessionId: number;
+    sessionEpoch: number;
+    topics: {
+        topicId: string;
+        partitions: {
+            partition: number;
+            currentLeaderEpoch: number;
+            fetchOffset: bigint;
+            lastFetchedEpoch: number;
+            logStartOffset: bigint;
+            partitionMaxBytes: number;
+        }[];
+    }[];
+    forgottenTopicsData: {
+        topicId: string;
+        partitions: number[];
+    }[];
+    rackId: string;
+}, {
+    responses: {
+        partitions: {
+            records: {
+                records: {
+                    attributes: number;
+                    timestampDelta: bigint;
+                    offsetDelta: number;
+                    key: Buffer | null;
+                    value: Buffer | null;
+                    headers: {
+                        key: Buffer | null;
+                        value: Buffer | null;
+                    }[];
+                }[];
+                compression: number;
+                timestampType: string;
+                isTransactional: boolean;
+                isControlBatch: boolean;
+                hasDeleteHorizonMs: boolean;
+                baseOffset: bigint;
+                batchLength: number;
+                partitionLeaderEpoch: number;
+                magic: number;
+                crc: number;
+                attributes: number;
+                lastOffsetDelta: number;
+                baseTimestamp: bigint;
+                maxTimestamp: bigint;
+                producerId: bigint;
+                producerEpoch: number;
+                baseSequence: number;
+            }[];
+            partitionIndex: number;
+            errorCode: number;
+            highWatermark: bigint;
+            lastStableOffset: bigint;
+            logStartOffset: bigint;
+            abortedTransactions: {
+                producerId: bigint;
+                firstOffset: bigint;
+                _tag: void;
+            }[];
+            preferredReadReplica: number;
+            _tag: void;
+        }[];
+        topicId: string;
+        _tag: void;
+    }[];
+    _tag: void;
+    throttleTimeMs: number;
+    errorCode: number;
+    sessionId: number;
+    _tag2: void;
+}>;

package/dist/api/fetch.js

@@ -0,0 +1,142 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FETCH = void 0;
+const codecs_1 = require("../codecs");
+const api_1 = require("../utils/api");
+const decoder_1 = require("../utils/decoder");
+const error_1 = require("../utils/error");
+exports.FETCH = (0, api_1.createApi)({
+    apiKey: 1,
+    apiVersion: 16,
+    request: (encoder, data) => encoder
+        .writeUVarInt(0)
+        .writeInt32(data.maxWaitMs)
+        .writeInt32(data.minBytes)
+        .writeInt32(data.maxBytes)
+        .writeInt8(data.isolationLevel)
+        .writeInt32(data.sessionId)
+        .writeInt32(data.sessionEpoch)
+        .writeCompactArray(data.topics, (encoder, topic) => encoder
+        .writeUUID(topic.topicId)
+        .writeCompactArray(topic.partitions, (encoder, partition) => encoder
+        .writeInt32(partition.partition)
+        .writeInt32(partition.currentLeaderEpoch)
+        .writeInt64(partition.fetchOffset)
+        .writeInt32(partition.lastFetchedEpoch)
+        .writeInt64(partition.logStartOffset)
+        .writeInt32(partition.partitionMaxBytes)
+        .writeUVarInt(0))
+        .writeUVarInt(0))
+        .writeCompactArray(data.forgottenTopicsData, (encoder, forgottenTopic) => encoder
+        .writeUUID(forgottenTopic.topicId)
+        .writeCompactArray(forgottenTopic.partitions, (encoder, partition) => encoder.writeInt32(partition))
+        .writeUVarInt(0))
+        .writeCompactString(data.rackId)
+        .writeUVarInt(0),
+    response: async (decoder) => {
+        const result = {
+            _tag: decoder.readTagBuffer(),
+            throttleTimeMs: decoder.readInt32(),
+            errorCode: decoder.readInt16(),
+            sessionId: decoder.readInt32(),
+            responses: decoder.readCompactArray((response) => ({
+                topicId: response.readUUID(),
+                partitions: response.readCompactArray((partition) => ({
+                    partitionIndex: partition.readInt32(),
+                    errorCode: partition.readInt16(),
+                    highWatermark: partition.readInt64(),
+                    lastStableOffset: partition.readInt64(),
+                    logStartOffset: partition.readInt64(),
+                    abortedTransactions: partition.readCompactArray((abortedTransaction) => ({
+                        producerId: abortedTransaction.readInt64(),
+                        firstOffset: abortedTransaction.readInt64(),
+                        _tag: abortedTransaction.readTagBuffer(),
+                    })),
+                    preferredReadReplica: partition.readInt32(),
+                    records: decodeRecordBatch(partition),
+                    _tag: partition.readTagBuffer(),
+                })),
+                _tag: response.readTagBuffer(),
+            })),
+            _tag2: decoder.readTagBuffer(),
+        };
+        if (result.errorCode)
+            throw new error_1.KafkaTSApiError(result.errorCode, null, result);
+        result.responses.forEach((response) => {
+            response.partitions.forEach((partition) => {
+                if (partition.errorCode)
+                    throw new error_1.KafkaTSApiError(partition.errorCode, null, result);
+            });
+        });
+        const decompressedResponses = await Promise.all(result.responses.map(async (response) => ({
+            ...response,
+            partitions: await Promise.all(response.partitions.map(async (partition) => ({
+                ...partition,
+                records: await Promise.all(partition.records.map(async ({ recordsLength, compressedRecords, ...record }) => {
+                    const { decompress } = (0, codecs_1.findCodec)(record.compression);
+                    const decompressedRecords = await decompress(compressedRecords);
+                    const decompressedDecoder = new decoder_1.Decoder(Buffer.concat([recordsLength, decompressedRecords]));
+                    return { ...record, records: decodeRecord(decompressedDecoder) };
+                })),
+            }))),
+        })));
+        return { ...result, responses: decompressedResponses };
+    },
+});
+const decodeRecordBatch = (decoder) => {
+    const size = decoder.readUVarInt() - 1;
+    if (size <= 0) {
+        return [];
+    }
+    const recordBatchDecoder = new decoder_1.Decoder(decoder.read(size));
+    const results = [];
+    while (recordBatchDecoder.getBufferLength() > recordBatchDecoder.getOffset() + 12) {
+        const baseOffset = recordBatchDecoder.readInt64();
+        const batchLength = recordBatchDecoder.readInt32();
+        if (!batchLength) {
+            continue;
+        }
+        const batchDecoder = new decoder_1.Decoder(recordBatchDecoder.read(batchLength));
+        const result = {
+            baseOffset,
+            batchLength,
+            partitionLeaderEpoch: batchDecoder.readInt32(),
+            magic: batchDecoder.readInt8(),
+            crc: batchDecoder.readUInt32(),
+            attributes: batchDecoder.readInt16(),
+            lastOffsetDelta: batchDecoder.readInt32(),
+            baseTimestamp: batchDecoder.readInt64(),
+            maxTimestamp: batchDecoder.readInt64(),
+            producerId: batchDecoder.readInt64(),
+            producerEpoch: batchDecoder.readInt16(),
+            baseSequence: batchDecoder.readInt32(),
+            recordsLength: batchDecoder.read(4),
+            compressedRecords: batchDecoder.read(),
+        };
+        const compression = result.attributes & 0x07;
+        const timestampType = (result.attributes & 0x08) >> 3 ? 'LogAppendTime' : 'CreateTime';
+        const isTransactional = !!((result.attributes & 0x10) >> 4);
+        const isControlBatch = !!((result.attributes & 0x20) >> 5);
+        const hasDeleteHorizonMs = !!((result.attributes & 0x40) >> 6);
+        results.push({
+            ...result,
+            compression,
+            timestampType,
+            isTransactional,
+            isControlBatch,
+            hasDeleteHorizonMs,
+        });
+    }
+    return results;
+};
+const decodeRecord = (decoder) => decoder.readRecords((record) => ({
+    attributes: record.readInt8(),
+    timestampDelta: record.readVarLong(),
+    offsetDelta: record.readVarInt(),
+    key: record.readVarIntBuffer(),
+    value: record.readVarIntBuffer(),
+    headers: record.readVarIntArray((header) => ({
+        key: header.readVarIntBuffer(),
+        value: header.readVarIntBuffer(),
+    })),
+}));

package/dist/api/find-coordinator.d.ts

@@ -0,0 +1,21 @@
+export declare const KEY_TYPE: {
+    GROUP: number;
+    TRANSACTION: number;
+};
+export declare const FIND_COORDINATOR: import("../utils/api").Api<{
+    keyType: number;
+    keys: string[];
+}, {
+    _tag: void;
+    throttleTimeMs: number;
+    coordinators: {
+        key: string | null;
+        nodeId: number;
+        host: string;
+        port: number;
+        errorCode: number;
+        errorMessage: string | null;
+        _tag: void;
+    }[];
+    _tag2: void;
+}>;

package/dist/api/find-coordinator.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FIND_COORDINATOR = exports.KEY_TYPE = void 0;
+const api_1 = require("../utils/api");
+const error_1 = require("../utils/error");
+exports.KEY_TYPE = {
+    GROUP: 0,
+    TRANSACTION: 1,
+};
+exports.FIND_COORDINATOR = (0, api_1.createApi)({
+    apiKey: 10,
+    apiVersion: 4,
+    request: (encoder, data) => encoder
+        .writeUVarInt(0)
+        .writeInt8(data.keyType)
+        .writeCompactArray(data.keys, (encoder, key) => encoder.writeCompactString(key))
+        .writeUVarInt(0),
+    response: (decoder) => {
+        const result = {
+            _tag: decoder.readTagBuffer(),
+            throttleTimeMs: decoder.readInt32(),
+            coordinators: decoder.readCompactArray((decoder) => ({
+                key: decoder.readCompactString(),
+                nodeId: decoder.readInt32(),
+                host: decoder.readCompactString(),
+                port: decoder.readInt32(),
+                errorCode: decoder.readInt16(),
+                errorMessage: decoder.readCompactString(),
+                _tag: decoder.readTagBuffer(),
+            })),
+            _tag2: decoder.readTagBuffer(),
+        };
+        result.coordinators.forEach((coordinator) => {
+            if (coordinator.errorCode)
+                throw new error_1.KafkaTSApiError(coordinator.errorCode, coordinator.errorMessage, result);
+        });
+        return result;
+    },
+});

package/dist/api/heartbeat.d.ts

@@ -0,0 +1,11 @@
+export declare const HEARTBEAT: import("../utils/api").Api<{
+    groupId: string;
+    generationId: number;
+    memberId: string;
+    groupInstanceId: string | null;
+}, {
+    _tag: void;
+    throttleTimeMs: number;
+    errorCode: number;
+    _tag2: void;
+}>;

package/dist/api/heartbeat.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HEARTBEAT = void 0;
+const api_1 = require("../utils/api");
+const error_1 = require("../utils/error");
+exports.HEARTBEAT = (0, api_1.createApi)({
+    apiKey: 12,
+    apiVersion: 4,
+    request: (encoder, data) => encoder
+        .writeUVarInt(0)
+        .writeCompactString(data.groupId)
+        .writeInt32(data.generationId)
+        .writeCompactString(data.memberId)
+        .writeCompactString(data.groupInstanceId)
+        .writeUVarInt(0),
+    response: (decoder) => {
+        const result = {
+            _tag: decoder.readTagBuffer(),
+            throttleTimeMs: decoder.readInt32(),
+            errorCode: decoder.readInt16(),
+            _tag2: decoder.readTagBuffer(),
+        };
+        if (result.errorCode)
+            throw new error_1.KafkaTSApiError(result.errorCode, null, result);
+        return result;
+    },
+});