kafka-ts 0.0.1-beta → 0.0.1-beta.1

package/.github/workflows/release.yml

@@ -0,0 +1,17 @@
+name: Publish package
+on:
+    release:
+        types: [published]
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: '20.x'
+                  registry-url: 'https://registry.npmjs.org'
+            - run: npm ci
+            - run: npm publish
+              env:
+                  NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}

package/.prettierrc

@@ -3,5 +3,6 @@
     "tabWidth": 4,
     "endOfLine": "lf",
     "semi": true,
-    "trailingComma": "all"
-}
+    "trailingComma": "all",
+    "singleQuote": true
+}

package/LICENSE

@@ -1,6 +1,6 @@
 The MIT License
 
-Copyright (c) 2018 Túlio Ornelas (ornelas.tulio@gmail.com)
+Copyright (c) 2024 Priit Käärd (priit@hashtatic.com)
 
 Permission is hereby granted, free of charge,
 to any person obtaining a copy of this software and

package/README.md

@@ -1,13 +1,13 @@
 # KafkaTS
 
-**Please note that this project is still in early development and is not yet ready for production use. The interface before stable release is subject to change.**
-
 **KafkaTS** is a Apache Kafka client library for Node.js. It provides both a low-level API for communicating directly with the Apache Kafka cluster and high-level APIs for publishing and subscribing to Kafka topics.
 
+**Please note that this project is still in early development and is not yet ready for production use. The interface before stable release is subject to change.**
+
 ## Installation
 
 ```bash
-npm install kafkats
+npm install kafka-ts
 ```
 
 ## Quick start
@@ -16,8 +16,8 @@ npm install kafkats
 
 ```typescript
 export const kafka = createKafkaClient({
-    clientId: "my-app",
-    bootstrapServers: [{ host: "localhost", port: 9092 }],
+    clientId: 'my-app',
+    bootstrapServers: [{ host: 'localhost', port: 9092 }],
 });
 ```
 
@@ -26,7 +26,7 @@ export const kafka = createKafkaClient({
 ```typescript
 const consumer = await kafka.startConsumer({
     groupId: 'my-consumer-group'.
-    topics: ["my-topic"],
+    topics: ['my-topic'],
     onMessage: (message) => {
         console.log(message);
     },
@@ -36,36 +36,47 @@ const consumer = await kafka.startConsumer({
 #### Producing messages
 
 ```typescript
-export const producer = kafka.createProcucer();
+export const producer = kafka.createProducer();
 
-await producer.send([{ topic: "example-topic-f", partition: 0, key: null, value: line }]);
+await producer.send([{ topic: 'my-topic', partition: 0, key: 'key', value: 'value' }]);
 ```
 
 #### Low-level API
 
 ```typescript
 const cluster = kafka.createCluster();
-    await cluster.connect();
-
-    const { controllerId } = await cluster.sendRequest(API.METADATA, {
-        allowTopicAutoCreation: false,
-        includeTopicAuthorizedOperations: false,
-        topics: [],
-    });
-
-    await cluster.sendRequestToNode(controllerId)(API.CREATE_TOPICS, {
-        validateOnly: false,
-        timeoutMs: 10_000,
-        topics: [
-            {
-                name: "my-topic",
-                numPartitions: 10,
-                replicationFactor: 3,
-                assignments: [],
-                configs: [],
-            },
-        ],
-    });
+await cluster.connect();
+
+const { controllerId } = await cluster.sendRequest(API.METADATA, {
+    allowTopicAutoCreation: false,
+    includeTopicAuthorizedOperations: false,
+    topics: [],
+});
+
+await cluster.sendRequestToNode(controllerId)(API.CREATE_TOPICS, {
+    validateOnly: false,
+    timeoutMs: 10_000,
+    topics: [
+        {
+            name: 'my-topic',
+            numPartitions: 10,
+            replicationFactor: 3,
+            assignments: [],
+            configs: [],
+        },
+    ],
+});
+
+await cluster.disconnect();
+```
+
+#### Graceful shutdown
+
+```typescript
+process.once('SIGTERM', async () => {
+    await consumer.close(); // waits for the consumer to finish processing the last batch and disconnects
+    await producer.close();
+});
 ```
 
 See the [examples](./examples) for more detailed examples.
@@ -77,12 +88,22 @@ The existing high-level libraries (e.g. kafkajs) are missing a few crucial featu
 
 ### New features compared to kafkajs
 
-* **Static consumer membership** - Rebalancing during rolling deployments cause delays. Using `groupInstanceId` in addition to `groupId` can avoid rebalancing and continue processing based on the existing assignment.
-* **Consuming messages without consumer groups** - When you don't need for the consumer to track the partition offsets, you can simply create a consumer without groupId and always either start consuming messages from the beginning or from the latest partition offset.
+-   **Static consumer membership** - Rebalancing during rolling deployments causes delays. Using `groupInstanceId` in addition to `groupId` can avoid rebalancing and continue consuming partitions in the existing assignment.
+-   **Consuming messages without consumer groups** - When you don't need the consumer to track the partition offsets, you can simply create a consumer without groupId and always either start consuming messages from the beginning or from the latest partition offset.
+-   **Low-level API requests** - It's possible to communicate directly with the Kafka cluster using the kafka api protocol.
+
+
+## Supported SASL mechanisms
+
+- PLAIN
+- SCRAM-SHA-256
+- SCRAM-SHA-512
+
+Custom SASL mechanisms can be implemented following the `SASLProvider` interface. See [src/auth](./src/auth) for examples.
 
 ## Backlog
 
-- [ ] Consumer concurrency control
-- [ ] Partitioner
-- [ ] API versioning
-- [ ] SCRAM support
+Minimal set of features left to implement before a stable release:
+
+-   Partitioner (Currently have to specify the partition on producer.send())
+-   API versioning (Currently only tested against Kafka 3.7+)

package/docker-compose.yml

@@ -1,104 +1,110 @@
 # kafka with raft:
 services:
-  kafka-0:
-    container_name: kafka-0
-    image: apache/kafka:3.7.1
-    ports:
-      - "9092:9092"
-      - "29092:29092"
-    environment:
-      KAFKA_NODE_ID: 0
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
-      KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
-      KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN'
-      KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9092,INTERBROKER://kafka-0:19092'
-      KAFKA_PROCESS_ROLES: 'broker,controller'
-      KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
-      KAFKA_LISTENERS: 'EXTERNAL://:9092,INTERBROKER://:19092,CONTROLLER://:29092'
-      KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
-      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
-      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
-      KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
-      KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
-      KAFKA_SSL_KEY_PASSWORD: 'password'
-      KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
-      KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password' 
-      KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: '' 
-      KAFKA_SSL_CLIENT_AUTH: 'required'
-    volumes:
-      - ./log4j.properties:/etc/kafka/docker/log4j.properties
-      - ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
-      - ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
-  kafka-1:
-    container_name: kafka-1
-    image: apache/kafka:3.7.1
-    ports:
-      - "9093:9093"
-      - "29093:29093"
-    environment:
-      KAFKA_NODE_ID: 1
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
-      KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
-      KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN'
-      KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9093,INTERBROKER://kafka-1:19093'
-      KAFKA_PROCESS_ROLES: 'broker,controller'
-      KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
-      KAFKA_LISTENERS: 'EXTERNAL://:9093,INTERBROKER://:19093,CONTROLLER://:29093'
-      KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
-      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
-      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
-      KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
-      KAFKA_SSL_KEYSTORE_PASSWORD: 'password' 
-      KAFKA_SSL_KEY_PASSWORD: 'password'
-      KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
-      KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password' 
-      KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: '' 
-      KAFKA_SSL_CLIENT_AUTH: 'required'
-    volumes:
-      - ./log4j.properties:/etc/kafka/docker/log4j.properties
-      - ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
-      - ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
-  kafka-2:
-    container_name: kafka-2
-    image: apache/kafka:3.7.1
-    ports:
-      - "9094:9094"
-      - "29094:29094"
-    environment:
-      KAFKA_NODE_ID: 2
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
-      KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
-      KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN'
-      KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9094,INTERBROKER://kafka-2:19094'
-      KAFKA_PROCESS_ROLES: 'broker,controller'
-      KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
-      KAFKA_LISTENERS: 'EXTERNAL://:9094,INTERBROKER://:19094,CONTROLLER://:29094'
-      KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
-      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
-      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
-      KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
-      KAFKA_SSL_KEYSTORE_PASSWORD: 'password' 
-      KAFKA_SSL_KEY_PASSWORD: 'password'
-      KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
-      KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password' 
-      KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: '' 
-      KAFKA_SSL_CLIENT_AUTH: 'required'
-    volumes:
-      - ./log4j.properties:/etc/kafka/docker/log4j.properties
-      - ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
-      - ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
+    kafka-0:
+        container_name: kafka-0
+        image: apache/kafka:3.7.1
+        ports:
+            - '9092:9092'
+            - '29092:29092'
+        environment:
+            KAFKA_NODE_ID: 0
+            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
+            KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
+            KAFKA_LISTENER_NAME_EXTERNAL_SCRAM-SHA-256_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.scram.ScramLoginModule required;'
+            KAFKA_LISTENER_NAME_EXTERNAL_SCRAM-SHA-512_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.scram.ScramLoginModule required;'
+            KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
+            KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9092,INTERBROKER://kafka-0:19092'
+            KAFKA_PROCESS_ROLES: 'broker,controller'
+            KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
+            KAFKA_LISTENERS: 'EXTERNAL://:9092,INTERBROKER://:19092,CONTROLLER://:29092'
+            KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
+            KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+            CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
+            KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+            KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+            KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+            KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+            KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
+            KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
+            KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
+            KAFKA_SSL_KEY_PASSWORD: 'password'
+            KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
+            KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password'
+            KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: ''
+            KAFKA_SSL_CLIENT_AUTH: 'required'
+        volumes:
+            - ./log4j.properties:/etc/kafka/docker/log4j.properties
+            - ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
+            - ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
+    kafka-1:
+        container_name: kafka-1
+        image: apache/kafka:3.7.1
+        ports:
+            - '9093:9093'
+            - '29093:29093'
+        environment:
+            KAFKA_NODE_ID: 1
+            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
+            KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
+            KAFKA_LISTENER_NAME_EXTERNAL_SCRAM-SHA-256_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.scram.ScramLoginModule required;'
+            KAFKA_LISTENER_NAME_EXTERNAL_SCRAM-SHA-512_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.scram.ScramLoginModule required;'
+            KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
+            KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9093,INTERBROKER://kafka-1:19093'
+            KAFKA_PROCESS_ROLES: 'broker,controller'
+            KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
+            KAFKA_LISTENERS: 'EXTERNAL://:9093,INTERBROKER://:19093,CONTROLLER://:29093'
+            KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
+            KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+            CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
+            KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+            KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+            KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+            KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+            KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
+            KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
+            KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
+            KAFKA_SSL_KEY_PASSWORD: 'password'
+            KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
+            KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password'
+            KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: ''
+            KAFKA_SSL_CLIENT_AUTH: 'required'
+        volumes:
+            - ./log4j.properties:/etc/kafka/docker/log4j.properties
+            - ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
+            - ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
+    kafka-2:
+        container_name: kafka-2
+        image: apache/kafka:3.7.1
+        ports:
+            - '9094:9094'
+            - '29094:29094'
+        environment:
+            KAFKA_NODE_ID: 2
+            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
+            KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
+            KAFKA_LISTENER_NAME_EXTERNAL_SCRAM-SHA-256_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.scram.ScramLoginModule required;'
+            KAFKA_LISTENER_NAME_EXTERNAL_SCRAM-SHA-512_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.scram.ScramLoginModule required;'
+            KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
+            KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9094,INTERBROKER://kafka-2:19094'
+            KAFKA_PROCESS_ROLES: 'broker,controller'
+            KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
+            KAFKA_LISTENERS: 'EXTERNAL://:9094,INTERBROKER://:19094,CONTROLLER://:29094'
+            KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
+            KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+            CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
+            KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+            KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+            KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+            KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+            KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
+            KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
+            KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
+            KAFKA_SSL_KEY_PASSWORD: 'password'
+            KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
+            KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password'
+            KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: ''
+            KAFKA_SSL_CLIENT_AUTH: 'required'
+        volumes:
+            - ./log4j.properties:/etc/kafka/docker/log4j.properties
+            - ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
+            - ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks

package/examples/package-lock.json

@@ -1,30 +1,31 @@
 {
-  "name": "examples",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "examples",
-      "version": "1.0.0",
-      "license": "ISC",
-      "dependencies": {
-        "kafkats": "file:../"
-      }
-    },
-    "..": {
-      "version": "1.0.0",
-      "license": "MIT",
-      "devDependencies": {
-        "@types/node": "^20.12.12",
-        "prettier": "^3.2.5",
-        "typescript": "^5.4.5",
-        "vitest": "^1.6.0"
-      }
-    },
-    "node_modules/kafkats": {
-      "resolved": "..",
-      "link": true
+    "name": "examples",
+    "version": "1.0.0",
+    "lockfileVersion": 3,
+    "requires": true,
+    "packages": {
+        "": {
+            "name": "examples",
+            "version": "1.0.0",
+            "license": "ISC",
+            "dependencies": {
+                "kafka-ts": "file:../"
+            }
+        },
+        "..": {
+            "name": "kafka-ts",
+            "version": "0.0.1-beta",
+            "license": "MIT",
+            "devDependencies": {
+                "@types/node": "^20.12.12",
+                "prettier": "^3.2.5",
+                "typescript": "^5.4.5",
+                "vitest": "^1.6.0"
+            }
+        },
+        "node_modules/kafka-ts": {
+            "resolved": "..",
+            "link": true
+        }
     }
-  }
 }

package/examples/package.json

@@ -1,14 +1,14 @@
 {
-  "name": "examples",
-  "version": "1.0.0",
-  "description": "",
-  "main": "dist/replicator.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "dependencies": {
-    "kafkats": "file:../"
-  },
-  "author": "",
-  "license": "ISC"
+    "name": "examples",
+    "version": "1.0.0",
+    "description": "",
+    "main": "dist/replicator.js",
+    "scripts": {
+        "test": "echo \"Error: no test specified\" && exit 1"
+    },
+    "dependencies": {
+        "kafka-ts": "file:../"
+    },
+    "author": "",
+    "license": "ISC"
 }

package/examples/src/client.ts

@@ -1,9 +1,9 @@
-import { readFileSync } from "fs";
-import { createKafkaClient } from "kafkats";
+import { readFileSync } from 'fs';
+import { createKafkaClient, saslScramSha512 } from 'kafka-ts';
 
 export const kafka = createKafkaClient({
-    clientId: "examples",
-    bootstrapServers: [{ host: "localhost", port: 9092 }],
-    sasl: { mechanism: "PLAIN", username: "admin", password: "admin" },
-    ssl: { ca: readFileSync("../certs/ca.crt").toString() },
+    clientId: 'examples',
+    bootstrapServers: [{ host: 'localhost', port: 9092 }],
+    sasl: saslScramSha512({ username: 'admin', password: 'admin' }),
+    ssl: { ca: readFileSync('../certs/ca.crt').toString() },
 });

package/examples/src/consumer.ts

@@ -1,17 +1,18 @@
-import { kafka } from "./client";
+import { kafka } from './client';
 
 (async () => {
     const consumer = await kafka.startConsumer({
-        groupId: "example-group",
-        groupInstanceId: "example-group-instance",
-        topics: ["example-topic-f"],
-        allowTopicAutoCreation: true,
-        onMessage: (message) => {
-            console.log(message);
+        groupId: 'example-group',
+        groupInstanceId: 'example-group-instance',
+        topics: ['my-topic'],
+        onBatch: (batch) => {
+            console.log(batch);
         },
+        granularity: 'broker',
+        concurrency: 10,
     });
 
-    process.on("SIGINT", async () => {
+    process.on('SIGINT', async () => {
         await consumer.close();
     });
 })();

package/examples/src/create-topic.ts

@@ -1,5 +1,5 @@
-import { kafka } from "./client";
-import { API } from "kafkats";
+import { API, API_ERROR, KafkaTSApiError } from 'kafka-ts';
+import { kafka } from './client';
 
 (async () => {
     const cluster = kafka.createCluster();
@@ -11,24 +11,30 @@ import { API } from "kafkats";
         topics: [],
     });
 
-    await cluster.sendRequestToNode(controllerId)(API.CREATE_TOPICS, {
-        validateOnly: false,
-        timeoutMs: 10_000,
-        topics: [
-            {
-                name: "my-topic",
-                numPartitions: 10,
-                replicationFactor: 3,
-                assignments: [],
-                configs: [],
-            },
-        ],
-    });
+    try {
+        await cluster.sendRequestToNode(controllerId)(API.CREATE_TOPICS, {
+            validateOnly: false,
+            timeoutMs: 10_000,
+            topics: [
+                {
+                    name: 'my-topic',
+                    numPartitions: 10,
+                    replicationFactor: 3,
+                    assignments: [],
+                    configs: [],
+                },
+            ],
+        });
+    } catch (error) {
+        if ((error as KafkaTSApiError).errorCode !== API_ERROR.TOPIC_ALREADY_EXISTS) {
+            throw error;
+        }
+    }
 
     const metadata = await cluster.sendRequestToNode(controllerId)(API.METADATA, {
         allowTopicAutoCreation: false,
         includeTopicAuthorizedOperations: false,
-        topics: [{ id: null, name: "my-topic" }],
+        topics: [{ id: null, name: 'my-topic' }],
     });
 
     console.log(metadata);

package/examples/src/producer.ts

@@ -1,24 +1,24 @@
-import { createInterface } from "readline";
-import { kafka } from "./client";
+import { createInterface } from 'readline';
+import { kafka } from './client';
 
 const producer = kafka.createProducer({ allowTopicAutoCreation: true });
 
 const rl = createInterface({ input: process.stdin, output: process.stdout });
 
-process.stdout.write("> ");
-rl.on("line", async (line) => {
+process.stdout.write('> ');
+rl.on('line', async (line) => {
     await producer.send([
         {
-            topic: "example-topic-f",
+            topic: 'example-topic-f',
             key: null,
             value: line,
             partition: 0,
         },
     ]);
-    process.stdout.write("> ");
+    process.stdout.write('> ');
 });
 
-process.on("SIGINT", async () => {
+process.on('SIGINT', async () => {
     rl.close();
     await producer.close();
 });

package/examples/src/replicator.ts

@@ -1,7 +1,7 @@
-import { kafka } from "./client";
+import { kafka } from './client';
 
 (async () => {
-    const topic = "example-topic";
+    const topic = 'example-topic';
 
     const producer = kafka.createProducer({ allowTopicAutoCreation: true });
     const consumer = await kafka.startConsumer({
@@ -10,7 +10,7 @@ import { kafka } from "./client";
             await producer.send(
                 messages.map((message) => ({
                     ...message,
-                    headers: { "X-Replicated": "true" },
+                    headers: { 'X-Replicated': 'true' },
                     topic: `${message.topic}-replicated`,
                     offset: 0n,
                 })),
@@ -18,7 +18,7 @@ import { kafka } from "./client";
             console.log(`Replicated ${messages.length} messages`);
         },
     });
-    process.on("SIGINT", async () => {
+    process.on('SIGINT', async () => {
         await consumer.close();
         await producer.close();
     });