kafka-ts 0.0.2-beta → 0.0.2

package/src/api/fetch.ts

@@ -1,12 +1,15 @@
-import { createApi } from "../utils/api";
-import { Decoder } from "../utils/decoder";
-import { KafkaTSApiError } from "../utils/error";
+import { findCodec } from '../codecs';
+import { createApi } from '../utils/api';
+import { Decoder } from '../utils/decoder';
+import { KafkaTSApiError } from '../utils/error';
 
 export const enum IsolationLevel {
     READ_UNCOMMITTED = 0,
     READ_COMMITTED = 1,
 }
 
+export type FetchResponse = Awaited<ReturnType<(typeof FETCH)['response']>>;
+
 export const FETCH = createApi({
     apiKey: 1,
     apiVersion: 16,
@@ -68,7 +71,7 @@ export const FETCH = createApi({
             )
             .writeCompactString(data.rackId)
             .writeUVarInt(0),
-    response: (decoder) => {
+    response: async (decoder) => {
         const result = {
             _tag: decoder.readTagBuffer(),
             throttleTimeMs: decoder.readInt32(),
@@ -88,56 +91,108 @@ export const FETCH = createApi({
                         _tag: abortedTransaction.readTagBuffer(),
                     })),
                     preferredReadReplica: partition.readInt32(),
-                    records: decodeRecords(partition),
+                    records: decodeRecordBatch(partition),
                     _tag: partition.readTagBuffer(),
                 })),
                 _tag: response.readTagBuffer(),
             })),
             _tag2: decoder.readTagBuffer(),
         };
+
         if (result.errorCode) throw new KafkaTSApiError(result.errorCode, null, result);
         result.responses.forEach((response) => {
             response.partitions.forEach((partition) => {
                 if (partition.errorCode) throw new KafkaTSApiError(partition.errorCode, null, result);
             });
         });
-        return result;
+
+        const decompressedResponses = await Promise.all(
+            result.responses.map(async (response) => ({
+                ...response,
+                partitions: await Promise.all(
+                    response.partitions.map(async (partition) => ({
+                        ...partition,
+                        records: await Promise.all(
+                            partition.records.map(async ({ recordsLength, compressedRecords, ...record }) => {
+                                const { decompress } = findCodec(record.compression);
+                                const decompressedRecords = await decompress(compressedRecords);
+                                const decompressedDecoder = new Decoder(
+                                    Buffer.concat([recordsLength, decompressedRecords]),
+                                );
+                                return { ...record, records: decodeRecord(decompressedDecoder) };
+                            }),
+                        ),
+                    })),
+                ),
+            })),
+        );
+
+        return { ...result, responses: decompressedResponses };
     },
 });
 
-const decodeRecords = (decoder: Decoder) => {
+const decodeRecordBatch = (decoder: Decoder) => {
     const size = decoder.readUVarInt() - 1;
     if (size <= 0) {
         return [];
     }
 
+    const recordBatchDecoder = new Decoder(decoder.read(size));
+
     const results = [];
-    while (decoder.getBufferLength() > decoder.getOffset() + 49) {
+    while (recordBatchDecoder.getBufferLength() > recordBatchDecoder.getOffset() + 12) {
+        const baseOffset = recordBatchDecoder.readInt64();
+        const batchLength = recordBatchDecoder.readInt32();
+        if (!batchLength) {
+            continue;
+        }
+
+        const batchDecoder = new Decoder(recordBatchDecoder.read(batchLength));
+
+        const result = {
+            baseOffset,
+            batchLength,
+            partitionLeaderEpoch: batchDecoder.readInt32(),
+            magic: batchDecoder.readInt8(),
+            crc: batchDecoder.readUInt32(),
+            attributes: batchDecoder.readInt16(),
+            lastOffsetDelta: batchDecoder.readInt32(),
+            baseTimestamp: batchDecoder.readInt64(),
+            maxTimestamp: batchDecoder.readInt64(),
+            producerId: batchDecoder.readInt64(),
+            producerEpoch: batchDecoder.readInt16(),
+            baseSequence: batchDecoder.readInt32(),
+            recordsLength: batchDecoder.read(4),
+            compressedRecords: batchDecoder.read(),
+        };
+
+        const compression = result.attributes & 0x07;
+        const timestampType = (result.attributes & 0x08) >> 3 ? 'LogAppendTime' : 'CreateTime';
+        const isTransactional = !!((result.attributes & 0x10) >> 4);
+        const isControlBatch = !!((result.attributes & 0x20) >> 5);
+        const hasDeleteHorizonMs = !!((result.attributes & 0x40) >> 6);
+
         results.push({
-            baseOffset: decoder.readInt64(),
-            batchLength: decoder.readInt32(),
-            partitionLeaderEpoch: decoder.readInt32(),
-            magic: decoder.readInt8(),
-            crc: decoder.readUInt32(),
-            attributes: decoder.readInt16(),
-            lastOffsetDelta: decoder.readInt32(),
-            baseTimestamp: decoder.readInt64(),
-            maxTimestamp: decoder.readInt64(),
-            producerId: decoder.readInt64(),
-            producerEpoch: decoder.readInt16(),
-            baseSequence: decoder.readInt32(),
-            records: decoder.readRecords((record) => ({
-                attributes: record.readInt8(),
-                timestampDelta: record.readVarLong(),
-                offsetDelta: record.readVarInt(),
-                key: record.readVarIntString(),
-                value: record.readVarIntString(),
-                headers: record.readCompactArray((header) => ({
-                    key: header.readVarIntString(),
-                    value: header.readVarIntString(),
-                })),
-            })),
+            ...result,
+            compression,
+            timestampType,
+            isTransactional,
+            isControlBatch,
+            hasDeleteHorizonMs,
         });
     }
     return results;
 };
+
+const decodeRecord = (decoder: Decoder) =>
+    decoder.readRecords((record) => ({
+        attributes: record.readInt8(),
+        timestampDelta: record.readVarLong(),
+        offsetDelta: record.readVarInt(),
+        key: record.readVarIntBuffer(),
+        value: record.readVarIntBuffer(),
+        headers: record.readVarIntArray((header) => ({
+            key: header.readVarIntBuffer(),
+            value: header.readVarIntBuffer(),
+        })),
+    }));

package/src/api/find-coordinator.ts

@@ -1,5 +1,5 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
 export const KEY_TYPE = {
     GROUP: 0,

package/src/api/heartbeat.ts

@@ -1,5 +1,5 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
 export const HEARTBEAT = createApi({
     apiKey: 12,

package/src/api/index.ts

@@ -1,21 +1,21 @@
-import { Api } from "../utils/api";
-import { API_VERSIONS } from "./api-versions";
-import { CREATE_TOPICS } from "./create-topics";
-import { DELETE_TOPICS } from "./delete-topics";
-import { FETCH } from "./fetch";
-import { FIND_COORDINATOR } from "./find-coordinator";
-import { HEARTBEAT } from "./heartbeat";
-import { INIT_PRODUCER_ID } from "./init-producer-id";
-import { JOIN_GROUP } from "./join-group";
-import { LEAVE_GROUP } from "./leave-group";
-import { LIST_OFFSETS } from "./list-offsets";
-import { METADATA } from "./metadata";
-import { OFFSET_COMMIT } from "./offset-commit";
-import { OFFSET_FETCH } from "./offset-fetch";
-import { PRODUCE } from "./produce";
-import { SASL_AUTHENTICATE } from "./sasl-authenticate";
-import { SASL_HANDSHAKE } from "./sasl-handshake";
-import { SYNC_GROUP } from "./sync-group";
+import { Api } from '../utils/api';
+import { API_VERSIONS } from './api-versions';
+import { CREATE_TOPICS } from './create-topics';
+import { DELETE_TOPICS } from './delete-topics';
+import { FETCH } from './fetch';
+import { FIND_COORDINATOR } from './find-coordinator';
+import { HEARTBEAT } from './heartbeat';
+import { INIT_PRODUCER_ID } from './init-producer-id';
+import { JOIN_GROUP } from './join-group';
+import { LEAVE_GROUP } from './leave-group';
+import { LIST_OFFSETS } from './list-offsets';
+import { METADATA } from './metadata';
+import { OFFSET_COMMIT } from './offset-commit';
+import { OFFSET_FETCH } from './offset-fetch';
+import { PRODUCE } from './produce';
+import { SASL_AUTHENTICATE } from './sasl-authenticate';
+import { SASL_HANDSHAKE } from './sasl-handshake';
+import { SYNC_GROUP } from './sync-group';
 
 export const API = {
     API_VERSIONS,
@@ -37,7 +37,9 @@ export const API = {
     SYNC_GROUP,
 };
 
-export const getApiName = (api: Api<unknown, unknown>) => Object.entries(API).find(([, v]) => v === api)?.[0];
+const apiNameByKey = Object.fromEntries(Object.entries(API).map(([k, v]) => [v.apiKey, k]));
+
+export const getApiName = <Request, Response>(api: Api<Request, Response>) => apiNameByKey[api.apiKey];
 
 export const API_ERROR = {
     UNKNOWN_SERVER_ERROR: -1,

package/src/api/init-producer-id.ts

@@ -1,5 +1,5 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
 export const INIT_PRODUCER_ID = createApi({
     apiKey: 22,

package/src/api/join-group.ts

@@ -1,6 +1,6 @@
-import { createApi } from "../utils/api";
-import { Encoder } from "../utils/encoder";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { Encoder } from '../utils/encoder';
+import { KafkaTSApiError } from '../utils/error';
 
 export const JOIN_GROUP = createApi({
     apiKey: 11,

package/src/api/leave-group.ts

@@ -1,5 +1,5 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
 export const LEAVE_GROUP = createApi({
     apiKey: 13,

package/src/api/list-offsets.ts

@@ -1,6 +1,6 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
-import { IsolationLevel } from "./fetch";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
+import { IsolationLevel } from './fetch';
 
 export const LIST_OFFSETS = createApi({
     apiKey: 2,

package/src/api/metadata.ts

@@ -1,7 +1,7 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
-export type Metadata = ReturnType<(typeof METADATA)["response"]>;
+export type Metadata = Awaited<ReturnType<(typeof METADATA)['response']>>;
 
 export const METADATA = createApi({
     apiKey: 3,

package/src/api/offset-commit.ts

@@ -1,5 +1,5 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
 export const OFFSET_COMMIT = createApi({
     apiKey: 8,

package/src/api/offset-fetch.ts

@@ -1,5 +1,5 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
 export const OFFSET_FETCH = createApi({
     apiKey: 9,

package/src/api/produce.ts

@@ -1,6 +1,6 @@
-import { createApi } from "../utils/api.js";
-import { Encoder } from "../utils/encoder.js";
-import { KafkaTSApiError } from "../utils/error.js";
+import { createApi } from '../utils/api.js';
+import { Encoder } from '../utils/encoder.js';
+import { KafkaTSApiError } from '../utils/error.js';
 
 export const PRODUCE = createApi({
     apiKey: 0,
@@ -28,11 +28,11 @@ export const PRODUCE = createApi({
                         attributes: number;
                         timestampDelta: bigint;
                         offsetDelta: number;
-                        key: string | null;
-                        value: string | null;
+                        key: Buffer | null;
+                        value: Buffer | null;
                         headers: {
-                            key: string;
-                            value: string;
+                            key: Buffer;
+                            value: Buffer;
                         }[];
                     }[];
                 }[];
@@ -61,14 +61,13 @@ export const PRODUCE = createApi({
                                     .writeInt8(record.attributes)
                                     .writeVarLong(record.timestampDelta)
                                     .writeVarInt(record.offsetDelta)
-                                    .writeVarIntString(record.key)
-                                    .writeVarIntString(record.value)
+                                    .writeVarIntBuffer(record.key)
+                                    .writeVarIntBuffer(record.value)
                                     .writeVarIntArray(record.headers, (encoder, header) =>
-                                        encoder.writeVarIntString(header.key).writeVarIntString(header.value),
-                                    )
-                                    .value();
+                                        encoder.writeVarIntBuffer(header.key).writeVarIntBuffer(header.value),
+                                    );
 
-                                return encoder.writeVarInt(recordBody.length).write(recordBody);
+                                return encoder.writeVarInt(recordBody.getByteLength()).writeEncoder(recordBody);
                             })
                             .value();
 
@@ -76,19 +75,17 @@ export const PRODUCE = createApi({
                             .writeInt32(partition.partitionLeaderEpoch)
                             .writeInt8(2) // magic byte
                             .writeUInt32(unsigned(crc32C(batchBody)))
-                            .write(batchBody)
-                            .value();
+                            .write(batchBody);
 
                         const batch = new Encoder()
                             .writeInt64(partition.baseOffset)
-                            .writeInt32(batchHeader.length)
-                            .write(batchHeader)
-                            .value();
+                            .writeInt32(batchHeader.getByteLength())
+                            .writeEncoder(batchHeader);
 
                         return encoder
                             .writeInt32(partition.index)
-                            .writeUVarInt(batch.length + 1) // batch size
-                            .write(batch)
+                            .writeUVarInt(batch.getByteLength() + 1) // batch size
+                            .writeEncoder(batch)
                             .writeUVarInt(0);
                     })
                     .writeUVarInt(0),

package/src/api/sasl-authenticate.ts

@@ -1,5 +1,5 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
 export const SASL_AUTHENTICATE = createApi({
     apiKey: 36,

package/src/api/sasl-handshake.ts

@@ -1,5 +1,5 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
 export const SASL_HANDSHAKE = createApi({
     apiKey: 17,

package/src/api/sync-group.ts

@@ -1,5 +1,5 @@
-import { createApi } from "../utils/api";
-import { KafkaTSApiError } from "../utils/error";
+import { createApi } from '../utils/api';
+import { KafkaTSApiError } from '../utils/error';
 
 export type Assignment = { [topic: string]: number[] };
 

package/src/auth/index.ts

@@ -0,0 +1,2 @@
+export { saslPlain } from './plain';
+export { saslScramSha256, saslScramSha512 } from './scram';

package/src/auth/plain.ts

@@ -0,0 +1,10 @@
+import { API } from "../api";
+import { SASLProvider } from "../broker";
+
+export const saslPlain = ({ username, password }: { username: string; password: string }): SASLProvider => ({
+    mechanism: 'PLAIN',
+    authenticate: async ({ sendRequest }) => {
+        const authBytes = [null, username, password].join('\u0000');
+        await sendRequest(API.SASL_AUTHENTICATE, { authBytes: Buffer.from(authBytes) });
+    },
+});

package/src/auth/scram.ts

@@ -0,0 +1,52 @@
+import { API } from '../api';
+import { SASLProvider } from '../broker';
+import { base64Decode, base64Encode, generateNonce, hash, hmac, saltPassword, xor } from '../utils/crypto';
+import { KafkaTSError } from '../utils/error';
+
+const saslScram =
+    ({ mechanism, keyLength, digest }: { mechanism: string; keyLength: number; digest: string }) =>
+    ({ username, password }: { username: string; password: string }): SASLProvider => ({
+        mechanism,
+        authenticate: async ({ sendRequest }) => {
+            const nonce = generateNonce();
+            const firstMessage = `n=${username},r=${nonce}`;
+
+            const { authBytes } = await sendRequest(API.SASL_AUTHENTICATE, {
+                authBytes: Buffer.from(`n,,${firstMessage}`),
+            });
+            if (!authBytes) {
+                throw new KafkaTSError('No auth response');
+            }
+
+            const response = Object.fromEntries(
+                authBytes
+                    .toString()
+                    .split(',')
+                    .map((pair) => pair.split('=')),
+            ) as { r: string; s: string; i: string };
+
+            const rnonce = response.r;
+            if (!rnonce.startsWith(nonce)) {
+                throw new KafkaTSError('Invalid nonce');
+            }
+            const iterations = parseInt(response.i);
+            const salt = base64Decode(response.s);
+
+            const saltedPassword = await saltPassword(password, salt, iterations, keyLength, digest);
+            const clientKey = hmac(saltedPassword, 'Client Key', digest);
+            const clientKeyHash = hash(clientKey, digest);
+
+            let finalMessage = `c=${base64Encode('n,,')},r=${rnonce}`;
+
+            const fullMessage = `${firstMessage},${authBytes.toString()},${finalMessage}`;
+            const clientSignature = hmac(clientKeyHash, fullMessage, digest);
+            const clientProof = base64Encode(xor(clientKey, clientSignature));
+
+            finalMessage += `,p=${clientProof}`;
+
+            await sendRequest(API.SASL_AUTHENTICATE, { authBytes: Buffer.from(finalMessage) });
+        },
+    });
+
+export const saslScramSha256 = saslScram({ mechanism: 'SCRAM-SHA-256', keyLength: 32, digest: 'sha256' });
+export const saslScramSha512 = saslScram({ mechanism: 'SCRAM-SHA-512', keyLength: 64, digest: 'sha512' });

package/src/broker.ts

@@ -1,16 +1,19 @@
-import { TcpSocketConnectOpts } from "net";
-import { TLSSocketOptions } from "tls";
-import { API } from "./api";
-import { Connection, SendRequest } from "./connection";
-import { KafkaTSError } from "./utils/error";
-import { memo } from "./utils/memo";
+import { TcpSocketConnectOpts } from 'net';
+import { TLSSocketOptions } from 'tls';
+import { API } from './api';
+import { Connection, SendRequest } from './connection';
+import { KafkaTSError } from './utils/error';
+import { memo } from './utils/memo';
 
-export type SASLOptions = { mechanism: "PLAIN"; username: string; password: string };
+export type SASLProvider = {
+    mechanism: string;
+    authenticate: (context: { sendRequest: SendRequest }) => Promise<void>;
+};
 
 type BrokerOptions = {
     clientId: string | null;
     options: TcpSocketConnectOpts;
-    sasl: SASLOptions | null;
+    sasl: SASLProvider | null;
     ssl: TLSSocketOptions | null;
 };
 
@@ -64,11 +67,6 @@ export class Broker {
     }
 
     private async saslAuthenticate() {
-        if (this.options.sasl?.mechanism !== "PLAIN") {
-            return;
-        }
-        const { username, password } = this.options.sasl;
-        const authBytes = [null, username, password].join("\u0000");
-        await this.sendRequest(API.SASL_AUTHENTICATE, { authBytes: Buffer.from(authBytes) });
+        await this.options.sasl?.authenticate({ sendRequest: this.sendRequest });
     }
 }

package/src/client.ts

@@ -1,14 +1,14 @@
-import { TcpSocketConnectOpts } from "net";
-import { TLSSocketOptions } from "tls";
-import { SASLOptions } from "./broker";
-import { Cluster } from "./cluster";
-import { Consumer, ConsumerOptions } from "./consumer/consumer";
-import { Producer, ProducerOptions } from "./producer/producer";
+import { TcpSocketConnectOpts } from 'net';
+import { TLSSocketOptions } from 'tls';
+import { SASLProvider } from './broker';
+import { Cluster } from './cluster';
+import { Consumer, ConsumerOptions } from './consumer/consumer';
+import { Producer, ProducerOptions } from './producer/producer';
 
 type ClientOptions = {
     clientId?: string | null;
     bootstrapServers: TcpSocketConnectOpts[];
-    sasl?: SASLOptions | null;
+    sasl?: SASLProvider | null;
     ssl?: TLSSocketOptions | null;
 };