kafka-ts 0.0.1-beta.3 → 0.0.1-beta.6

package/examples/package.json

@@ -5,10 +5,17 @@
     "main": "dist/replicator.js",
     "scripts": {
         "test": "echo \"Error: no test specified\" && exit 1",
+        "start:jaeger": "docker run --rm --name jaeger -e COLLECTOR_OTLP_ENABLED=true -p 16686:16686 -p 4317:4317 -p 4318:4318 -d jaegertracing/all-in-one:1.62.0",
         "build": "tsc"
     },
     "dependencies": {
-        "kafka-ts": "file:../"
+        "@opentelemetry/api": "^1.9.0",
+        "@opentelemetry/auto-instrumentations-node": "^0.51.0",
+        "@opentelemetry/context-async-hooks": "^1.27.0",
+        "@opentelemetry/exporter-trace-otlp-grpc": "^0.54.0",
+        "@opentelemetry/sdk-node": "^0.54.0",
+        "kafka-ts": "file:../",
+        "kafkajs": "^2.2.4"
     },
     "author": "",
     "license": "ISC"

package/examples/src/benchmark/common.ts

@@ -0,0 +1,98 @@
+import { delay } from '../utils/delay';
+
+export const startBenchmarker = async ({
+    createTopic,
+    connectProducer,
+    startConsumer,
+    produce,
+}: {
+    createTopic: (opts: { topic: string; partitions: number; replicationFactor: number }) => Promise<void>;
+    connectProducer: () => Promise<() => unknown>;
+    startConsumer: (
+        opts: {
+            groupId: string;
+            topic: string;
+            concurrency: number;
+            incrementCount: (key: string, value: number) => void;
+        },
+        callback: (timestamp: number) => void,
+    ) => Promise<() => unknown>;
+    produce: (opts: { topic: string; length: number; timestamp: number; acks: -1 | 1 }) => Promise<void>;
+}) => {
+    const benchmarkId = `benchmark-${Date.now()}`;
+    const {
+        TOPIC = benchmarkId,
+        PRODUCER = 'true',
+        CONSUMER = 'true',
+        PARTITIONS = '10',
+        REPLICATION_FACTOR = '3',
+        CONCURRENCY = '1',
+        PRODUCE_BATCH_SIZE = '10',
+        PRODUCE_DELAY_MS = '0',
+    } = process.env;
+    const enableProducer = PRODUCER === 'true';
+    const enableConsumer = CONSUMER === 'true';
+    const partitions = parseInt(PARTITIONS);
+    const replicationFactor = parseInt(REPLICATION_FACTOR);
+    const concurrency = parseInt(CONCURRENCY);
+    const produceBatchSize = parseInt(PRODUCE_BATCH_SIZE);
+    const produceDelayMs = parseInt(PRODUCE_DELAY_MS);
+
+    await createTopic({ topic: TOPIC, partitions, replicationFactor }).catch(console.error);
+    await delay(2500);
+
+    let counts: Record<string, number> = {};
+    let sums: Record<string, number> = {};
+
+    const incrementCount = (key: string, value: number) => {
+        counts[key] = (counts[key] || 0) + value;
+    };
+
+    const incrementSum = (key: string, value: number) => {
+        sums[key] = (sums[key] || 0) + value;
+    };
+
+    const stopProducer = await connectProducer();
+
+    const stopConsumer =
+        enableConsumer &&
+        (await startConsumer({ groupId: benchmarkId, topic: TOPIC, concurrency, incrementCount }, (timestamp) => {
+            incrementCount('CONSUMER', 1);
+            incrementSum('CONSUMER', Date.now() - timestamp);
+        }));
+
+    const interval = setInterval(() => {
+        const latencies = Object.entries(sums)
+            .map(([key, sum]) => `${key} ${(sum / counts[key]).toFixed(2)}ms`)
+            .sort()
+            .join(', ');
+
+        const counters = Object.entries(counts)
+            .map(([key, count]) => `${key} ${count}`)
+            .sort()
+            .join(', ');
+
+        console.log(`Latency: ${latencies} | Counters: ${counters}`);
+        counts = {};
+        sums = {};
+    }, 1000);
+
+    let isRunning = true;
+    const produceLoop = async () => {
+        if (!isRunning) return;
+        const start = Date.now();
+        await produce({ topic: TOPIC, length: produceBatchSize, timestamp: Date.now(), acks: -1 });
+        incrementCount('PRODUCER', 1);
+        incrementSum('PRODUCER', Date.now() - start);
+        produceDelayMs && (await delay(produceDelayMs));
+        produceLoop();
+    };
+    enableProducer && produceLoop();
+
+    process.once('SIGINT', async () => {
+        isRunning = false;
+        stopConsumer && (await stopConsumer());
+        await stopProducer();
+        clearInterval(interval);
+    });
+};

package/examples/src/benchmark/kafka-ts.ts

@@ -0,0 +1,67 @@
+import { readFileSync } from 'fs';
+import { API, createKafkaClient, saslScramSha512 } from 'kafka-ts';
+import { startBenchmarker } from './common';
+
+// setTracer(new OpenTelemetryTracer());
+
+const kafka = createKafkaClient({
+    bootstrapServers: [{ host: 'localhost', port: 9092 }],
+    clientId: 'kafka-ts',
+    sasl: saslScramSha512({ username: 'admin', password: 'admin' }),
+    ssl: { ca: readFileSync('../certs/ca.crt').toString() },
+});
+
+const producer = kafka.createProducer({ allowTopicAutoCreation: false });
+
+startBenchmarker({
+    createTopic: async ({ topic, partitions, replicationFactor }) => {
+        const cluster = kafka.createCluster();
+        await cluster.connect();
+
+        const { controllerId } = await cluster.sendRequest(API.METADATA, {
+            allowTopicAutoCreation: false,
+            includeTopicAuthorizedOperations: false,
+            topics: [],
+        });
+        await cluster.setSeedBroker(controllerId);
+        await cluster.sendRequest(API.CREATE_TOPICS, {
+            validateOnly: false,
+            timeoutMs: 10_000,
+            topics: [
+                {
+                    name: topic,
+                    numPartitions: partitions,
+                    replicationFactor,
+                    assignments: [],
+                    configs: [],
+                },
+            ],
+        });
+        await cluster.disconnect();
+    },
+    connectProducer: async () => () => producer.close(),
+    startConsumer: async ({ groupId, topic, concurrency, incrementCount }, callback) => {
+        const consumer = await kafka.startConsumer({
+            groupId,
+            topics: [topic],
+            onBatch: async (messages) => {
+                for (const message of messages) {
+                    callback(parseInt(message.timestamp.toString()));
+                }
+            },
+            concurrency,
+        });
+        consumer.on('offsetCommit', () => incrementCount('OFFSET_COMMIT', 1));
+        return () => consumer.close();
+    },
+    produce: async ({ topic, length, timestamp, acks }) => {
+        await producer.send(
+            Array.from({ length }).map(() => ({
+                topic: topic,
+                value: Buffer.from('hello'),
+                timestamp: BigInt(timestamp),
+            })),
+            { acks },
+        );
+    },
+});

package/examples/src/benchmark/kafkajs.ts

@@ -0,0 +1,51 @@
+import { readFileSync } from 'fs';
+import { Kafka } from 'kafkajs';
+import { startBenchmarker } from './common';
+
+const kafkajs = new Kafka({
+    brokers: ['localhost:9092'],
+    clientId: 'kafkajs',
+    sasl: { username: 'admin', password: 'admin', mechanism: 'plain' },
+    ssl: { ca: readFileSync('../certs/ca.crt').toString() },
+});
+
+const producer = kafkajs.producer({ allowAutoTopicCreation: false });
+
+startBenchmarker({
+    createTopic: async ({ topic, partitions, replicationFactor }) => {
+        const admin = kafkajs.admin();
+        await admin.connect();
+        await admin.createTopics({ topics: [{ topic, numPartitions: partitions, replicationFactor }] });
+        await admin.disconnect();
+    },
+    connectProducer: async () => {
+        await producer.connect();
+        return () => producer.disconnect();
+    },
+    startConsumer: async ({ groupId, topic, concurrency, incrementCount }, callback) => {
+        const consumer = kafkajs.consumer({ groupId, allowAutoTopicCreation: false });
+        await consumer.connect();
+        await consumer.subscribe({ topic });
+        await consumer.run({
+            eachBatch: async ({ batch }) => {
+                for (const message of batch.messages) {
+                    callback(parseInt(message.timestamp));
+                }
+            },
+            partitionsConsumedConcurrently: concurrency,
+            autoCommit: true,
+        });
+        consumer.on(consumer.events.COMMIT_OFFSETS, () => incrementCount('OFFSET_COMMIT', 1));
+        return () => consumer.disconnect();
+    },
+    produce: async ({ topic, length, timestamp, acks }) => {
+        await producer.send({
+            topic,
+            messages: Array.from({ length }).map(() => ({
+                value: Buffer.from(timestamp.toString()),
+                timestamp: timestamp.toString(),
+            })),
+            acks,
+        });
+    },
+});

package/examples/src/client.ts

@@ -1,5 +1,8 @@
 import { readFileSync } from 'fs';
-import { createKafkaClient, saslScramSha512 } from 'kafka-ts';
+import { createKafkaClient, saslScramSha512, setTracer } from 'kafka-ts';
+import { OpenTelemetryTracer } from './opentelemetry';
+
+setTracer(new OpenTelemetryTracer());
 
 export const kafka = createKafkaClient({
     clientId: 'examples',

package/examples/src/consumer.ts

@@ -1,12 +1,18 @@
+import { jsonSerializer, log } from 'kafka-ts';
 import { kafka } from './client';
+import { delay } from '../../dist/utils/delay';
 
 (async () => {
     const consumer = await kafka.startConsumer({
         groupId: 'example-group',
         groupInstanceId: 'example-group-instance',
         topics: ['my-topic'],
+        allowTopicAutoCreation: true,
         onBatch: (batch) => {
-            console.log(batch);
+            log.info(
+                `Received batch: ${JSON.stringify(batch.map((message) => ({ ...message, value: message.value?.toString() })), jsonSerializer)}`,
+            );
+            log.info(`Latency: ${Date.now() - parseInt(batch[0].timestamp.toString())}ms`)
         },
         batchGranularity: 'broker',
         concurrency: 10,

package/examples/src/create-topic.ts

@@ -1,4 +1,4 @@
-import { API, API_ERROR, KafkaTSApiError } from 'kafka-ts';
+import { API, API_ERROR, KafkaTSApiError, log } from 'kafka-ts';
 import { kafka } from './client';
 
 (async () => {
@@ -19,7 +19,7 @@ import { kafka } from './client';
                 {
                     name: 'my-topic',
                     numPartitions: 10,
-                    replicationFactor: 3,
+                    replicationFactor: 1,
                     assignments: [],
                     configs: [],
                 },
@@ -37,7 +37,7 @@ import { kafka } from './client';
         topics: [{ id: null, name: 'my-topic' }],
     });
 
-    console.log(metadata);
+    log.info('Metadata', metadata);
 
     await cluster.disconnect();
 })();

package/examples/src/opentelemetry.ts

@@ -0,0 +1,46 @@
+import { context, ROOT_CONTEXT, trace } from '@opentelemetry/api';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { AsyncHooksContextManager } from '@opentelemetry/context-async-hooks';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import { Tracer } from 'kafka-ts';
+
+const contextManager = new AsyncHooksContextManager();
+contextManager.enable();
+context.setGlobalContextManager(contextManager);
+
+const exporter = new OTLPTraceExporter({ url: 'http://localhost:4317' });
+
+const sdk = new NodeSDK({
+    serviceName: 'kafka-ts',
+    traceExporter: exporter,
+    spanProcessors: [new BatchSpanProcessor(exporter)],
+    instrumentations: [getNodeAutoInstrumentations()],
+});
+
+sdk.start();
+
+process.once('SIGINT', () => {
+    sdk.shutdown();
+});
+
+const tracer = trace.getTracer('kafka-ts');
+
+export class OpenTelemetryTracer implements Tracer {
+    startActiveSpan(module, method, { body, ...metadata } = {} as any, callback) {
+        return tracer.startActiveSpan(
+            `${module}.${method} ${metadata?.message ?? ''}`,
+            { attributes: metadata },
+            metadata?.root ? ROOT_CONTEXT : context.active(),
+            (span) => {
+                const result = callback();
+                if (result instanceof Promise) {
+                    return result.finally(() => span.end());
+                }
+                span.end();
+                return result;
+            },
+        );
+    }
+}

package/examples/src/producer.ts

@@ -4,19 +4,19 @@ import { kafka } from './client';
 const producer = kafka.createProducer({ allowTopicAutoCreation: true });
 
 const rl = createInterface({ input: process.stdin, output: process.stdout });
+process.once('SIGINT', rl.close);
 
 process.stdout.write('> ');
 rl.on('line', async (line) => {
-    await producer.send([
-        {
-            topic: 'example-topic-f',
-            value: Buffer.from(line),
-        },
-    ]);
+    await producer.send(
+        [
+            {
+                topic: 'my-topic',
+                value: Buffer.from(line),
+            },
+        ],
+        { acks: -1 },
+    );
     process.stdout.write('> ');
 });
-
-process.on('SIGINT', async () => {
-    rl.close();
-    await producer.close();
-});
+rl.once('close', () => producer.close());

package/examples/src/replicator.ts

@@ -1,3 +1,4 @@
+import { log } from 'kafka-ts';
 import { kafka } from './client';
 
 (async () => {
@@ -15,7 +16,7 @@ import { kafka } from './client';
                     offset: 0n,
                 })),
             );
-            console.log(`Replicated ${messages.length} messages`);
+            log.info(`Replicated ${messages.length} messages`);
         },
     });
     process.on('SIGINT', async () => {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.1-beta.3",
+    "version": "0.0.1-beta.6",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",
@@ -9,7 +9,9 @@
         "url": "https://github.com/priitkaard/kafka-ts.git"
     },
     "scripts": {
-        "version:bump": "npm version prerelease --preid=beta",
+        "start": "docker-compose down && KAFKA_VERSION=3.7.1 docker-compose up -d && sleep 5 && bash ./scripts/create-scram-user.sh",
+        "version:beta": "npm version prerelease --preid=beta",
+        "version:patch": "npm version patch",
         "format": "prettier --write .",
         "build": "tsc",
         "watch": "tsc -w",

package/scripts/create-scram-user.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 set -e
 
-kafka-configs --bootstrap-server localhost:9092 --command-config kafka-local.properties --alter --add-config 'SCRAM-SHA-256=[password=admin]' --entity-type users --entity-name admin
-kafka-configs --bootstrap-server localhost:9092 --command-config kafka-local.properties --alter --add-config 'SCRAM-SHA-512=[password=admin]' --entity-type users --entity-name admin
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+
+kafka-configs --bootstrap-server localhost:9092 --command-config "$SCRIPT_DIR/kafka-local.properties" --alter --add-config 'SCRAM-SHA-256=[password=admin]' --entity-type users --entity-name admin
+kafka-configs --bootstrap-server localhost:9092 --command-config "$SCRIPT_DIR/kafka-local.properties" --alter --add-config 'SCRAM-SHA-512=[password=admin]' --entity-type users --entity-name admin

package/scripts/generate-certs.sh

@@ -23,3 +23,5 @@ keytool -importkeystore -srckeystore certs/kafka.keystore.jks \
     -deststorepass password \
     -destkeystore certs/kafka.truststore.jks \
     -noprompt
+
+rm certs/{ca.key,ca.srl,kafka.crt,kafka.csr,kafka.key}