@relaymesh/relaybus-kafka 0.0.4 → 0.0.6

package/README.md

@@ -10,6 +10,8 @@ npm install @relaymesh/relaybus-kafka
 
 ## Example
 
+Publisher will attempt to create the topic on first publish (requires broker permissions).
+
 ```ts
 import { KafkaPublisher, KafkaSubscriber } from "@relaymesh/relaybus-kafka";
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@relaymesh/relaybus-kafka",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
@@ -8,7 +8,6 @@
     "test": "vitest run"
   },
   "dependencies": {
-    "@relaymesh/relaybus-core": "workspace:*",
     "kafkajs": "^2.2.4"
   }
 }

package/src/core.ts

@@ -0,0 +1,203 @@
+import { randomUUID } from "node:crypto";
+
+export type DecodedMessage = {
+  v: "v1";
+  id: string;
+  topic: string;
+  ts: Date;
+  contentType: string;
+  payload: Buffer;
+  meta: Record<string, string>;
+};
+
+export type OutgoingMessage = {
+  id?: string;
+  topic: string;
+  ts?: Date;
+  contentType?: string;
+  payload: Buffer | Uint8Array;
+  meta?: Record<string, string>;
+  v?: "v1";
+};
+
+export type NormalizeOptions = {
+  now?: () => Date;
+  idGenerator?: () => string;
+};
+
+type EnvelopeJSON = {
+  v: string;
+  id: string;
+  topic: string;
+  ts: string;
+  content_type: string;
+  payload_b64: string;
+  meta: Record<string, string>;
+};
+
+type NormalizedMessage = {
+  id: string;
+  topic: string;
+  ts: Date;
+  contentType: string;
+  payload: Buffer;
+  meta: Record<string, string>;
+  v: "v1";
+};
+
+const DEFAULT_CONTENT_TYPE = "application/octet-stream";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function assertStringField(value: unknown, field: string): string {
+  if (typeof value !== "string" || value.length === 0) {
+    throw new Error(`invalid ${field}`);
+  }
+  return value;
+}
+
+function assertMeta(value: unknown): Record<string, string> {
+  if (!isRecord(value)) {
+    throw new Error("invalid meta");
+  }
+  const meta: Record<string, string> = {};
+  for (const [k, v] of Object.entries(value)) {
+    if (typeof v !== "string") {
+      throw new Error("invalid meta");
+    }
+    meta[k] = v;
+  }
+  return meta;
+}
+
+function normalizeMessage(message: OutgoingMessage, options?: NormalizeOptions): NormalizedMessage {
+  if (!message) {
+    throw new Error("message is required");
+  }
+  if (!message.topic) {
+    throw new Error("topic is required");
+  }
+  if (message.payload === undefined || message.payload === null) {
+    throw new Error("payload is required");
+  }
+  const payload = Buffer.isBuffer(message.payload)
+    ? message.payload
+    : Buffer.from(message.payload);
+
+  const now = options?.now ?? (() => new Date());
+  const idGenerator = options?.idGenerator ?? randomUUID;
+  const id = message.id && message.id.length > 0 ? message.id : idGenerator();
+  if (!id) {
+    throw new Error("id is required");
+  }
+  const ts = message.ts ?? now();
+  if (!(ts instanceof Date) || Number.isNaN(ts.getTime())) {
+    throw new Error("invalid ts");
+  }
+
+  const meta: Record<string, string> = {};
+  if (message.meta) {
+    for (const [k, v] of Object.entries(message.meta)) {
+      if (typeof v !== "string") {
+        throw new Error("invalid meta");
+      }
+      meta[k] = v;
+    }
+  }
+
+  const contentType = message.contentType && message.contentType.length > 0
+    ? message.contentType
+    : DEFAULT_CONTENT_TYPE;
+
+  return {
+    id,
+    topic: message.topic,
+    ts,
+    contentType,
+    payload,
+    meta,
+    v: "v1"
+  };
+}
+
+function isValidBase64(value: string): boolean {
+  if (value === "") {
+    return true;
+  }
+  if (value.length % 4 !== 0) {
+    return false;
+  }
+  if (!/^[A-Za-z0-9+/]+={0,2}$/.test(value)) {
+    return false;
+  }
+  const decoded = Buffer.from(value, "base64");
+  return decoded.toString("base64") === value;
+}
+
+export function decodeEnvelope(jsonBytes: Buffer | string): DecodedMessage {
+  const raw = Buffer.isBuffer(jsonBytes) ? jsonBytes.toString("utf8") : jsonBytes;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error("invalid json");
+  }
+  if (!isRecord(parsed)) {
+    throw new Error("invalid envelope");
+  }
+
+  const env = parsed as EnvelopeJSON;
+  if (env.v !== "v1") {
+    throw new Error("invalid v");
+  }
+
+  const id = assertStringField(env.id, "id");
+  const topic = assertStringField(env.topic, "topic");
+  const tsRaw = assertStringField(env.ts, "ts");
+  const contentType = assertStringField(env.content_type, "content_type");
+
+  if (typeof env.payload_b64 !== "string") {
+    throw new Error("invalid payload_b64");
+  }
+  if (!isValidBase64(env.payload_b64)) {
+    throw new Error("invalid payload_b64");
+  }
+  const payload = Buffer.from(env.payload_b64, "base64");
+
+  const meta = assertMeta(env.meta);
+  const ts = new Date(tsRaw);
+  if (Number.isNaN(ts.getTime())) {
+    throw new Error("invalid ts");
+  }
+
+  return {
+    v: "v1",
+    id,
+    topic,
+    ts,
+    contentType,
+    payload,
+    meta
+  };
+}
+
+export function encodeEnvelope(message: OutgoingMessage, options?: NormalizeOptions): Buffer {
+  const normalized = normalizeMessage(message, options);
+  const envelope: EnvelopeJSON = {
+    v: normalized.v,
+    id: normalized.id,
+    topic: normalized.topic,
+    ts: normalized.ts.toISOString(),
+    content_type: normalized.contentType,
+    payload_b64: normalized.payload.toString("base64"),
+    meta: normalized.meta
+  };
+
+  return Buffer.from(JSON.stringify(envelope), "utf8");
+}
+
+export const defaults = {
+  DEFAULT_CONTENT_TYPE
+};

package/src/index.ts

@@ -1,5 +1,5 @@
-import { decodeEnvelope, encodeEnvelope, DecodedMessage, OutgoingMessage } from "@relaymesh/relaybus-core";
-import { Kafka, Consumer, Producer } from "kafkajs";
+import { decodeEnvelope, encodeEnvelope, DecodedMessage, OutgoingMessage } from "./core";
+import { Kafka, Consumer, Producer, Admin } from "kafkajs";
 
 export type KafkaProducer = {
   send: (record: { topic: string; key?: Buffer; value: Buffer }) => Promise<void> | void;
@@ -8,6 +8,7 @@ export type KafkaProducer = {
 export type KafkaPublisherConfig = {
   producer: KafkaProducer;
   topicPrefix?: string;
+  admin?: Admin;
 };
 
 export type KafkaPublisherConnectConfig = {
@@ -20,6 +21,9 @@ export class KafkaPublisher {
   private readonly producer: KafkaProducer;
   private readonly prefix: string;
   private rawProducer?: Producer;
+  private admin?: Admin;
+  private ownsAdmin = false;
+  private readonly ensuredTopics = new Set<string>();
 
   constructor(config: KafkaPublisherConfig) {
     if (!config.producer) {
@@ -27,6 +31,8 @@ export class KafkaPublisher {
     }
     this.producer = config.producer;
     this.prefix = config.topicPrefix ?? "";
+    this.admin = config.admin;
+    this.ownsAdmin = false;
   }
 
   static async connect(config: KafkaPublisherConnectConfig): Promise<KafkaPublisher> {
@@ -35,7 +41,9 @@ export class KafkaPublisher {
       brokers: config.brokers
     });
     const producer = kafka.producer();
+    const admin = kafka.admin();
     await producer.connect();
+    await admin.connect();
     const publisher = new KafkaPublisher({
       producer: {
         send: async (record) => {
@@ -48,6 +56,8 @@ export class KafkaPublisher {
       topicPrefix: config.topicPrefix
     });
     publisher.rawProducer = producer;
+    publisher.admin = admin;
+    publisher.ownsAdmin = true;
     return publisher;
   }
 
@@ -59,6 +69,7 @@ export class KafkaPublisher {
       key: message.id ? Buffer.from(message.id) : undefined,
       value: payload
     };
+    await this.ensureTopic(record.topic);
     await Promise.resolve(this.producer.send(record));
   }
 
@@ -66,6 +77,31 @@ export class KafkaPublisher {
     if (this.rawProducer) {
       await this.rawProducer.disconnect();
     }
+    if (this.admin && this.ownsAdmin) {
+      await this.admin.disconnect();
+    }
+  }
+
+  private async ensureTopic(topic: string): Promise<void> {
+    if (!this.admin) {
+      return;
+    }
+    if (this.ensuredTopics.has(topic)) {
+      return;
+    }
+    try {
+      await this.admin.createTopics({
+        topics: [{ topic, numPartitions: 1, replicationFactor: 1 }],
+        waitForLeaders: true
+      });
+      this.ensuredTopics.add(topic);
+    } catch (err) {
+      if (isTopicAlreadyExists(err)) {
+        this.ensuredTopics.add(topic);
+        return;
+      }
+      // Best-effort provisioning: allow publish to proceed even if create fails.
+    }
   }
 }
 
@@ -179,3 +215,21 @@ function resolveTopic(argumentTopic: string, messageTopic?: string): string {
   }
   return topic;
 }
+
+function isTopicAlreadyExists(err: unknown): boolean {
+  if (!err || typeof err !== "object") {
+    return false;
+  }
+  const record = err as {
+    type?: string;
+    name?: string;
+    topicErrors?: Array<{ error?: { type?: string } }>;
+  };
+  if (record.type === "TOPIC_ALREADY_EXISTS") {
+    return true;
+  }
+  if (record.name === "KafkaJSCreateTopicError" && Array.isArray(record.topicErrors)) {
+    return record.topicErrors.every((topicError) => topicError?.error?.type === "TOPIC_ALREADY_EXISTS");
+  }
+  return false;
+}

package/test/kafka.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { KafkaPublisher, KafkaSubscriber } from "../src/index";
-import { decodeEnvelope } from "@relaymesh/relaybus-core";
+import { decodeEnvelope } from "../src/core";
 
 describe("KafkaPublisher", () => {
   it("sends encoded envelope", async () => {

package/tsconfig.json

@@ -4,11 +4,7 @@
     "rootDir": "src",
     "outDir": "dist",
     "declaration": true,
-    "types": ["node"],
-    "baseUrl": ".",
-    "paths": {
-      "@relaymesh/relaybus-core": ["../../core/typescript/dist"]
-    }
+    "types": ["node"]
   },
   "include": ["src/**/*.ts"]
 }