@relaymesh/relaybus-kafka 0.0.2

package/README.md

@@ -0,0 +1,35 @@
+# relaybus-kafka (TypeScript)
+
+Kafka publisher and subscriber utilities for Relaybus.
+
+## Install
+
+```
+npm install @relaymesh/relaybus-kafka
+```
+
+## Example
+
+```ts
+import { KafkaPublisher, KafkaSubscriber } from "@relaymesh/relaybus-kafka";
+
+async function main() {
+  const publisher = await KafkaPublisher.connect({
+    brokers: ["localhost:29092"]
+  });
+  await publisher.publish("relaybus.demo", {
+    topic: "relaybus.demo",
+    payload: Buffer.from("hello")
+  });
+  await publisher.close();
+
+  const subscriber = await KafkaSubscriber.connect({
+    brokers: ["localhost:29092"],
+    groupId: "relaybus",
+    onMessage: (msg) => console.log(msg.topic, msg.payload.toString())
+  });
+  await subscriber.start("relaybus.demo");
+}
+
+main().catch(console.error);
+```

package/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@relaymesh/relaybus-kafka",
+  "version": "0.0.2",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@relaymesh/relaybus-core": "workspace:*",
+    "kafkajs": "^2.2.4"
+  }
+}

package/src/index.ts

@@ -0,0 +1,181 @@
+import { decodeEnvelope, encodeEnvelope, DecodedMessage, OutgoingMessage } from "@relaybus/relaybus-core";
+import { Kafka, Consumer, Producer } from "kafkajs";
+
+export type KafkaProducer = {
+  send: (record: { topic: string; key?: Buffer; value: Buffer }) => Promise<void> | void;
+};
+
+export type KafkaPublisherConfig = {
+  producer: KafkaProducer;
+  topicPrefix?: string;
+};
+
+export type KafkaPublisherConnectConfig = {
+  brokers: string[];
+  topicPrefix?: string;
+  clientId?: string;
+};
+
+export class KafkaPublisher {
+  private readonly producer: KafkaProducer;
+  private readonly prefix: string;
+  private rawProducer?: Producer;
+
+  constructor(config: KafkaPublisherConfig) {
+    if (!config.producer) {
+      throw new Error("producer is required");
+    }
+    this.producer = config.producer;
+    this.prefix = config.topicPrefix ?? "";
+  }
+
+  static async connect(config: KafkaPublisherConnectConfig): Promise<KafkaPublisher> {
+    const kafka = new Kafka({
+      clientId: config.clientId ?? "relaybus",
+      brokers: config.brokers
+    });
+    const producer = kafka.producer();
+    await producer.connect();
+    const publisher = new KafkaPublisher({
+      producer: {
+        send: async (record) => {
+          await producer.send({
+            topic: record.topic,
+            messages: [{ key: record.key, value: record.value }]
+          });
+        }
+      },
+      topicPrefix: config.topicPrefix
+    });
+    publisher.rawProducer = producer;
+    return publisher;
+  }
+
+  async publish(topic: string, message: OutgoingMessage): Promise<void> {
+    const resolved = resolveTopic(topic, message.topic);
+    const payload = encodeEnvelope({ ...message, topic: resolved });
+    const record = {
+      topic: `${this.prefix}${resolved}`,
+      key: message.id ? Buffer.from(message.id) : undefined,
+      value: payload
+    };
+    await Promise.resolve(this.producer.send(record));
+  }
+
+  async close(): Promise<void> {
+    if (this.rawProducer) {
+      await this.rawProducer.disconnect();
+    }
+  }
+}
+
+export type KafkaSubscriberConfig = {
+  onMessage: (msg: DecodedMessage) => void | Promise<void>;
+};
+
+export type KafkaSubscriberConnectConfig = {
+  brokers: string[];
+  onMessage: (msg: DecodedMessage) => void | Promise<void>;
+  groupId?: string;
+  clientId?: string;
+  maxMessages?: number;
+};
+
+export class KafkaSubscriber {
+  private readonly onMessage: (msg: DecodedMessage) => void | Promise<void>;
+  private consumer?: Consumer;
+  private maxMessages?: number;
+
+  constructor(config: KafkaSubscriberConfig) {
+    this.onMessage = config.onMessage;
+  }
+
+  static async connect(config: KafkaSubscriberConnectConfig): Promise<KafkaSubscriber> {
+    const kafka = new Kafka({
+      clientId: config.clientId ?? "relaybus",
+      brokers: config.brokers
+    });
+    const consumer = kafka.consumer({ groupId: config.groupId ?? "relaybus" });
+    await consumer.connect();
+    const subscriber = new KafkaSubscriber({ onMessage: config.onMessage });
+    subscriber.consumer = consumer;
+    subscriber.maxMessages = config.maxMessages ?? 1;
+    return subscriber;
+  }
+
+  async handleMessage(data: Buffer | string): Promise<void> {
+    const decoded = decodeEnvelope(data);
+    await this.onMessage(decoded);
+  }
+
+  async start(topic: string): Promise<void> {
+    if (!this.consumer) {
+      throw new Error("consumer is not initialized");
+    }
+    await this.consumer.subscribe({ topic, fromBeginning: true });
+    const consumer = this.consumer;
+    const max = this.maxMessages ?? 1;
+    let count = 0;
+    let finished = false;
+
+    let finishError: Error | undefined;
+    const finish = async (err?: Error) => {
+      if (finished) {
+        return;
+      }
+      finished = true;
+      if (err) {
+        finishError = err;
+      }
+      try {
+        await consumer.stop();
+      } catch {
+        // Ignore stop failures.
+      }
+      try {
+        await consumer.disconnect();
+      } catch {
+        // Ignore disconnect failures.
+      }
+    };
+
+    try {
+      await consumer.run({
+        eachMessage: async ({ message }) => {
+          const value = message.value ? Buffer.from(message.value) : Buffer.from("");
+          await this.handleMessage(value);
+          count++;
+          if (max > 0 && count >= max) {
+            await finish();
+          }
+        }
+      });
+    } catch (err) {
+      await finish(err as Error);
+    }
+    if (finishError) {
+      throw finishError;
+    }
+  }
+
+  async close(): Promise<void> {
+    if (this.consumer) {
+      try {
+        await this.consumer.disconnect();
+      } catch {
+        // Ignore double-disconnects.
+      }
+    }
+  }
+}
+
+function resolveTopic(argumentTopic: string, messageTopic?: string): string {
+  const topic = messageTopic && messageTopic.length > 0 ? messageTopic : argumentTopic;
+  if (!topic) {
+    throw new Error("topic is required");
+  }
+  if (argumentTopic && messageTopic && argumentTopic !== messageTopic) {
+    throw new Error(`topic mismatch: ${messageTopic} vs ${argumentTopic}`);
+  }
+  return topic;
+}

package/test/kafka.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from "vitest";
+import { KafkaPublisher, KafkaSubscriber } from "../src/index";
+import { decodeEnvelope } from "@relaybus/relaybus-core";
+
+describe("KafkaPublisher", () => {
+  it("sends encoded envelope", async () => {
+    const records: any[] = [];
+    const publisher = new KafkaPublisher({
+      producer: {
+        send: async (record) => {
+          records.push(record);
+        }
+      },
+      topicPrefix: "rb-"
+    });
+
+    await publisher.publish("alpha", {
+      id: "id-1",
+      topic: "alpha",
+      payload: Buffer.from("hi")
+    });
+
+    expect(records).toHaveLength(1);
+    expect(records[0].topic).toBe("rb-alpha");
+    const decoded = decodeEnvelope(records[0].value);
+    expect(decoded.id).toBe("id-1");
+  });
+});
+
+describe("KafkaSubscriber", () => {
+  it("decodes envelope", async () => {
+    let seen = "";
+    const subscriber = new KafkaSubscriber({
+      onMessage: (msg) => {
+        seen = msg.id;
+      }
+    });
+
+    await subscriber.handleMessage(
+      Buffer.from(
+        JSON.stringify({
+          v: "v1",
+          id: "id",
+          topic: "alpha",
+          ts: "2024-01-01T00:00:00Z",
+          content_type: "text/plain",
+          payload_b64: "aGVsbG8=",
+          meta: {}
+        })
+      )
+    );
+
+    expect(seen).toBe("id");
+  });
+
+  it("rejects missing fields", async () => {
+    const subscriber = new KafkaSubscriber({ onMessage: () => {} });
+    await expect(subscriber.handleMessage("{}"))
+      .rejects.toThrow(/invalid v|invalid id/);
+  });
+});

package/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "declaration": true,
+    "types": ["node"],
+    "baseUrl": ".",
+    "paths": {
+      "@relaybus/relaybus-core": ["../../core/typescript/dist"]
+    }
+  },
+  "include": ["src/**/*.ts"]
+}