@beespoke/micro-server-kafka 0.0.0-stage → 2.0.3

package/README.md

@@ -1,3 +1,97 @@
-# Temporary Holding Version
+# @beespoke/micro-server-kafka
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+KafkaJS producers, consumers, topic helpers, and application adapter discovery.
+
+Supports Node 24+ and Bun 1.3.14+.
+
+## Install
+
+```sh
+npm install @beespoke/micro-server @beespoke/micro-server-kafka
+```
+
+Use `bun add` in place of `npm install` for Bun projects.
+
+## Configure the plugin
+
+```ts
+import { defineConfig } from '@beespoke/micro-server';
+
+export default defineConfig({
+    plugins: ['@beespoke/micro-server-kafka'],
+    pluginOptions: {
+        '@beespoke/micro-server-kafka': {
+            config: {
+                clientId: 'my-api',
+                brokers: (process.env.KAFKA_BROKERS || '127.0.0.1:9092').split(','),
+            },
+            consumerPath: './kafka/consumers',
+            producerPath: './kafka/producers',
+        },
+    },
+});
+```
+
+Create both discovery directories before starting:
+
+```sh
+mkdir -p kafka/consumers kafka/producers
+```
+
+Requires reachable Kafka brokers. `config` accepts KafkaJS options such as TLS, SASL, retries, and timeouts. Optional `rootDir` changes the application discovery root. Discovered classes must follow `ServeKafka`'s `_`, `connect`, and `init` adapter conventions; bare `BaseConsumer` and `BaseProducer` classes are not automatically configured adapters.
+
+## Standalone producer
+
+For explicit application-owned clients, HTTP core is optional:
+
+```ts
+import { BaseKafka, BaseProducer } from '@beespoke/micro-server-kafka';
+
+const kafka = new BaseKafka({
+    clientId: 'worker',
+    brokers: ['127.0.0.1:9092'],
+});
+const producer = new BaseProducer(kafka);
+await producer.connect();
+try {
+    await producer.publish({
+        topic: 'orders', event: 'created', message: { orderId: '123' },
+    });
+} finally {
+    await producer.disconnect();
+}
+```
+
+Root exports include `BaseKafka`, `BaseProducer`, `BaseConsumer`, `ServeKafka`, `ServeKafkaOptions`, and message/publish types. The plugin owns cleanup of clients acquired through discovery; standalone applications own their clients' connect/disconnect lifecycle.
+
+## Start the application
+
+Save the configuration as `micro-server.config.ts` next to your application's `package.json`, then create `index.ts`:
+
+```ts
+import { ApplicationServer } from '@beespoke/micro-server';
+
+await ApplicationServer.bootstrap();
+for (const signal of ['SIGINT', 'SIGTERM'] as const) {
+    process.once(signal, () => {
+        void ApplicationServer.close().catch(error => {
+            console.error(error);
+            process.exitCode = 1;
+        });
+    });
+}
+```
+
+Run `bun index.ts` or `node --env-file-if-exists=.env index.ts`. Node requires ESM (`"type": "module"` in package.json). When using Bun, load configuration secrets from the application environment or its `.env` file.
+
+## TypeScript and lifecycle
+
+Add this type-only import to an application-owned `plugin-types.d.ts` for configuration autocomplete:
+
+```ts
+import type {} from '@beespoke/micro-server-kafka/plugin';
+```
+
+Core validates plugin options before setup and owns cleanup after successful startup or startup failure. Await `ApplicationServer.bootstrap()` before using the configured client, and call `ApplicationServer.close()` during application shutdown. Importing the package does not open a connection. Relative plugin paths resolve from the selected configuration file's directory.
+
+The root export can also be used without HTTP core. In that case, explicitly initialize and close the client. The `/plugin` entry is the integration adapter for `@beespoke/micro-server`.

package/dist/BaseConsumer.d.ts

@@ -0,0 +1,14 @@
+import type { ConsumerRunConfig, Consumer, ConsumerConfig } from 'kafkajs';
+import type { BaseKafka } from './BaseKafka.js';
+import type { MessageHandler } from './BrokerType.js';
+export declare class BaseConsumer {
+    consumer: Consumer | null;
+    private readonly kafka;
+    private connecting?;
+    private closing?;
+    private connected;
+    constructor(baseKafka: BaseKafka);
+    connect(groupId: string, config?: ConsumerConfig): Promise<void>;
+    disconnect(): Promise<void>;
+    subscribe(topic: string, messageHandler: MessageHandler, consumerRunConfig?: ConsumerRunConfig): Promise<void>;
+}

package/dist/BaseKafka.d.ts

@@ -0,0 +1,11 @@
+import { Kafka, type KafkaConfig } from 'kafkajs';
+export declare class BaseKafka {
+    private readonly kafka;
+    constructor(config: KafkaConfig);
+    getKafka(): Kafka;
+    createTopics(topics: {
+        topic: string;
+        partitions: number;
+        replicationFactor: number;
+    }[]): Promise<void>;
+}

package/dist/BaseProducer.d.ts

@@ -0,0 +1,14 @@
+import { type ProducerConfig } from 'kafkajs';
+import type { BaseKafka } from './BaseKafka.js';
+import type { PublishType } from './BrokerType.js';
+export declare class BaseProducer {
+    private producer;
+    private readonly kafka;
+    private connecting?;
+    private closing?;
+    private connected;
+    constructor(baseKafka: BaseKafka);
+    connect(config?: ProducerConfig): Promise<void>;
+    disconnect(): Promise<void>;
+    publish(data: PublishType): Promise<boolean>;
+}

package/dist/BrokerType.d.ts

@@ -0,0 +1,12 @@
+export type PublishType = {
+    topic: string;
+    headers?: Record<string, string>;
+    event: string;
+    message: Record<string, any>;
+};
+export type MessageType = {
+    headers?: Record<string, string>;
+    event: string;
+    data: any;
+};
+export type MessageHandler = (message: MessageType) => Promise<void>;

package/dist/PluginContext.d.ts

@@ -0,0 +1,13 @@
+/** Structural subset of the core context; standalone clients need no HTTP dependency. */
+export interface Logger {
+    info(...args: unknown[]): void;
+    warn(...args: unknown[]): void;
+    error(...args: unknown[]): void;
+    debug(...args: unknown[]): void;
+}
+export interface PluginContext {
+    readonly logger: Logger;
+    readonly configDir: string;
+    onStart(hook: () => void | Promise<void>): void;
+    onClose(hook: () => void | Promise<void>): void;
+}

package/dist/ServeKafka.d.ts

@@ -0,0 +1,25 @@
+import type { KafkaConfig } from 'kafkajs';
+import type { PluginContext, Logger } from './PluginContext.js';
+export interface ServeKafkaOptions {
+    /** Explicit root takes precedence over the application's config directory. */
+    rootDir?: string;
+    logger?: Logger;
+}
+/** Discovers existing consumer/producer classes, including their static `_` facade. */
+export declare class ServeKafka {
+    private readonly config;
+    private readonly consumerPath;
+    private readonly producerPath;
+    private readonly options;
+    readonly name = "kafka";
+    private rootDir;
+    private logger;
+    private readonly resources;
+    private cleanupFailed;
+    constructor(config: KafkaConfig, consumerPath?: string, producerPath?: string, options?: ServeKafkaOptions);
+    setup(context: PluginContext): Promise<void>;
+    startConsumers(): Promise<void>;
+    startProducers(): Promise<void>;
+    private start;
+    close(): Promise<void>;
+}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { BaseKafka } from './BaseKafka.js';
+export { BaseConsumer } from './BaseConsumer.js';
+export { BaseProducer } from './BaseProducer.js';
+export { ServeKafka } from './ServeKafka.js';
+export type { ServeKafkaOptions } from './ServeKafka.js';
+export type { PublishType, MessageType, MessageHandler } from './BrokerType.js';

package/dist/index.js

@@ -0,0 +1,286 @@
+// src/BaseKafka.ts
+import { Kafka, logLevel } from "kafkajs";
+
+class BaseKafka {
+  kafka;
+  constructor(config) {
+    this.kafka = new Kafka({ ...config, logLevel: config.logLevel ?? logLevel.INFO });
+  }
+  getKafka() {
+    return this.kafka;
+  }
+  async createTopics(topics) {
+    const admin = this.kafka.admin();
+    let failure;
+    let failed = false;
+    try {
+      await admin.connect();
+      const existingTopics = await admin.listTopics();
+      for (const { topic, partitions, replicationFactor } of topics) {
+        if (!existingTopics.includes(topic)) {
+          await admin.createTopics({ topics: [{ topic, numPartitions: partitions, replicationFactor }] });
+        }
+      }
+    } catch (error) {
+      failure = error;
+      failed = true;
+    }
+    try {
+      await admin.disconnect();
+    } catch (error) {
+      if (failed)
+        throw new AggregateError([failure, error], "Kafka topic creation and admin cleanup failed", { cause: failure });
+      throw error;
+    }
+    if (failed)
+      throw failure;
+  }
+}
+// src/BaseConsumer.ts
+class BaseConsumer {
+  consumer = null;
+  kafka;
+  connecting;
+  closing;
+  connected = false;
+  constructor(baseKafka) {
+    this.kafka = baseKafka.getKafka();
+  }
+  async connect(groupId, config) {
+    if (this.closing)
+      await this.closing;
+    if (this.connecting)
+      return this.connecting;
+    if (this.connected)
+      return;
+    if (this.consumer)
+      await this.disconnect();
+    const consumer = this.kafka.consumer({ groupId, ...config });
+    this.consumer = consumer;
+    this.connecting = (async () => {
+      try {
+        await consumer.connect();
+        this.connected = true;
+      } catch (error) {
+        try {
+          await consumer.disconnect();
+          this.consumer = null;
+        } catch (cleanupError) {
+          throw new AggregateError([error, cleanupError], "Kafka consumer connection and cleanup failed", { cause: error });
+        }
+        throw error;
+      }
+    })();
+    try {
+      await this.connecting;
+    } finally {
+      this.connecting = undefined;
+    }
+  }
+  async disconnect() {
+    if (this.closing)
+      return this.closing;
+    this.closing = (async () => {
+      try {
+        await this.connecting;
+      } catch {}
+      const consumer = this.consumer;
+      this.connected = false;
+      if (consumer) {
+        await consumer.disconnect();
+        this.consumer = null;
+      }
+    })();
+    try {
+      await this.closing;
+    } finally {
+      this.closing = undefined;
+    }
+  }
+  async subscribe(topic, messageHandler, consumerRunConfig) {
+    const consumer = this.consumer;
+    if (!this.connected || !consumer)
+      throw new Error("Consumer not connected");
+    await consumer.subscribe({ topic, fromBeginning: true });
+    await consumer.run({
+      ...consumerRunConfig,
+      eachMessage: async ({ topic: topic2, partition, message }) => {
+        const parsedMessage = {
+          headers: message.headers,
+          event: message.key?.toString() || "",
+          data: message.value ? JSON.parse(message.value.toString()) : null
+        };
+        await messageHandler(parsedMessage);
+        await consumer.commitOffsets([{ topic: topic2, partition, offset: (BigInt(message.offset) + 1n).toString() }]);
+      }
+    });
+  }
+}
+// src/BaseProducer.ts
+import { Partitioners } from "kafkajs";
+
+class BaseProducer {
+  producer = null;
+  kafka;
+  connecting;
+  closing;
+  connected = false;
+  constructor(baseKafka) {
+    this.kafka = baseKafka.getKafka();
+  }
+  async connect(config) {
+    if (this.closing)
+      await this.closing;
+    if (this.connecting)
+      return this.connecting;
+    if (this.connected)
+      return;
+    if (this.producer)
+      await this.disconnect();
+    const producer = this.kafka.producer({ createPartitioner: Partitioners.DefaultPartitioner, ...config });
+    this.producer = producer;
+    this.connecting = (async () => {
+      try {
+        await producer.connect();
+        this.connected = true;
+      } catch (error) {
+        try {
+          await producer.disconnect();
+          this.producer = null;
+        } catch (cleanupError) {
+          throw new AggregateError([error, cleanupError], "Kafka producer connection and cleanup failed", { cause: error });
+        }
+        throw error;
+      }
+    })();
+    try {
+      await this.connecting;
+    } finally {
+      this.connecting = undefined;
+    }
+  }
+  async disconnect() {
+    if (this.closing)
+      return this.closing;
+    this.closing = (async () => {
+      try {
+        await this.connecting;
+      } catch {}
+      const producer = this.producer;
+      this.connected = false;
+      if (producer) {
+        await producer.disconnect();
+        this.producer = null;
+      }
+    })();
+    try {
+      await this.closing;
+    } finally {
+      this.closing = undefined;
+    }
+  }
+  async publish(data) {
+    if (!this.connected || !this.producer)
+      throw new Error("Producer not connected");
+    const result = await this.producer.send({
+      topic: data.topic,
+      messages: [{ headers: data.headers, key: data.event, value: JSON.stringify(data.message) }]
+    });
+    return result.length > 0;
+  }
+}
+// src/ServeKafka.ts
+import { readdir } from "node:fs/promises";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+
+class ServeKafka {
+  config;
+  consumerPath;
+  producerPath;
+  options;
+  name = "kafka";
+  rootDir;
+  logger;
+  resources = new Map;
+  cleanupFailed = false;
+  constructor(config, consumerPath = "kafka/consumers", producerPath = "kafka/producers", options = {}) {
+    this.config = config;
+    this.consumerPath = consumerPath;
+    this.producerPath = producerPath;
+    this.options = options;
+    this.rootDir = resolve(options.rootDir ?? process.cwd());
+    this.logger = options.logger ?? console;
+  }
+  async setup(context) {
+    this.rootDir = resolve(this.options.rootDir ?? context.configDir);
+    this.logger = this.options.logger ?? context.logger;
+    context.onClose(() => this.close());
+    await this.startConsumers();
+    await this.startProducers();
+  }
+  async startConsumers() {
+    await this.start(this.consumerPath, "consumer");
+  }
+  async startProducers() {
+    await this.start(this.producerPath, "producer");
+  }
+  async start(directory, kind) {
+    if (this.cleanupFailed)
+      throw new Error("Kafka adapter cleanup failed; await close() successfully before restarting");
+    try {
+      const fullDirectory = resolve(this.rootDir, directory);
+      const files = (await readdir(fullDirectory)).filter((file) => new RegExp(`\\.${kind}\\.(?:ts|mts|js|mjs)$`).test(file)).sort();
+      for (const file of files) {
+        const filePath = resolve(fullDirectory, file);
+        if (this.resources.has(filePath))
+          continue;
+        const AdapterClass = (await import(pathToFileURL(filePath).href)).default;
+        if (typeof AdapterClass !== "function")
+          throw new Error(`Kafka adapter ${file} must default-export a class`);
+        const instance = new AdapterClass(this.config);
+        const client = AdapterClass._ ?? instance._ ?? instance;
+        if (typeof client.connect !== "function" || typeof client.disconnect !== "function") {
+          throw new Error(`Kafka adapter ${file} must expose connect() and disconnect(), directly or through static _`);
+        }
+        this.resources.set(filePath, client);
+        const baseName = file.replace(new RegExp(`\\.${kind}\\.(?:ts|mts|js|mjs)$`), "");
+        await client.connect(kind === "consumer" ? `${baseName}-group` : this.config.clientId);
+        if (kind === "consumer") {
+          if (typeof AdapterClass.init === "function")
+            await AdapterClass.init();
+          else if (typeof instance.init === "function")
+            await instance.init();
+        }
+        this.logger.info(`Started ${kind}: ${baseName}`);
+      }
+    } catch (error) {
+      try {
+        await this.close();
+      } catch (cleanupError) {
+        throw new AggregateError([error, cleanupError], "Kafka startup and cleanup failed", { cause: error });
+      }
+      throw error;
+    }
+  }
+  async close() {
+    const errors = [];
+    for (const [key, client] of [...this.resources].reverse()) {
+      try {
+        await client.disconnect();
+        this.resources.delete(key);
+      } catch (error) {
+        errors.push(error);
+      }
+    }
+    this.cleanupFailed = errors.length > 0;
+    if (errors.length)
+      throw new AggregateError(errors, "Failed to close Kafka adapters");
+  }
+}
+export {
+  ServeKafka,
+  BaseProducer,
+  BaseKafka,
+  BaseConsumer
+};

package/dist/plugin/index.d.ts

@@ -0,0 +1,15 @@
+import type { PluginDefinition } from '@beespoke/micro-server';
+import type { KafkaConfig } from 'kafkajs';
+export interface KafkaPluginOptions {
+    config: KafkaConfig;
+    consumerPath?: string;
+    producerPath?: string;
+    rootDir?: string;
+}
+declare module '@beespoke/micro-server' {
+    interface PluginOptionsRegistry {
+        '@beespoke/micro-server-kafka': KafkaPluginOptions;
+    }
+}
+declare const definition: PluginDefinition<KafkaPluginOptions>;
+export default definition;

package/dist/plugin/index.js

@@ -0,0 +1,88 @@
+// src/plugin/index.ts
+import { ServeKafka } from "../index.js";
+function object(value) {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}
+var definition = {
+  name: "@beespoke/micro-server-kafka",
+  validate(options) {
+    if (!object(options))
+      throw new Error("Kafka options must be an object");
+    if (Object.keys(options).some((key) => !["config", "consumerPath", "producerPath", "rootDir"].includes(key)))
+      throw new Error("Kafka options contain an unknown field");
+    for (const key of ["consumerPath", "producerPath", "rootDir"]) {
+      if (options[key] !== undefined && (typeof options[key] !== "string" || !options[key].trim()))
+        throw new Error(`Kafka ${key} must be a nonempty string`);
+    }
+    const config = options.config;
+    if (!object(config))
+      throw new Error("Kafka config must be an object");
+    const configKeys = ["brokers", "ssl", "sasl", "clientId", "connectionTimeout", "authenticationTimeout", "reauthenticationThreshold", "requestTimeout", "enforceRequestTimeout", "retry", "socketFactory", "logLevel", "logCreator"];
+    if (Object.keys(config).some((key) => !configKeys.includes(key)))
+      throw new Error("Kafka config contains an unknown field");
+    if (typeof config.brokers !== "function" && (!Array.isArray(config.brokers) || !config.brokers.length || config.brokers.some((broker) => typeof broker !== "string" || !broker.trim())))
+      throw new Error("Kafka config.brokers must be a nonempty string array or function");
+    if (config.clientId !== undefined && (typeof config.clientId !== "string" || !config.clientId.trim()))
+      throw new Error("Kafka config.clientId must be a nonempty string");
+    for (const key of ["connectionTimeout", "authenticationTimeout", "reauthenticationThreshold", "requestTimeout"]) {
+      if (config[key] !== undefined && (typeof config[key] !== "number" || !Number.isFinite(config[key]) || config[key] < 0))
+        throw new Error(`Kafka config.${key} must be a nonnegative number`);
+    }
+    for (const key of ["socketFactory", "logCreator"]) {
+      if (config[key] !== undefined && typeof config[key] !== "function")
+        throw new Error(`Kafka config.${key} must be a function`);
+    }
+    if (config.enforceRequestTimeout !== undefined && typeof config.enforceRequestTimeout !== "boolean")
+      throw new Error("Kafka config.enforceRequestTimeout must be a boolean");
+    if (config.logLevel !== undefined && ![0, 1, 2, 4, 5].includes(config.logLevel))
+      throw new Error("Kafka config.logLevel must be a Kafka log level");
+    if (config.ssl !== undefined && typeof config.ssl !== "boolean" && !object(config.ssl))
+      throw new Error("Kafka config.ssl must be a boolean or TLS options object");
+    if (config.retry !== undefined) {
+      if (!object(config.retry))
+        throw new Error("Kafka config.retry must be an object");
+      const retryKeys = ["maxRetryTime", "initialRetryTime", "factor", "multiplier", "retries", "restartOnFailure"];
+      if (Object.keys(config.retry).some((key) => !retryKeys.includes(key)))
+        throw new Error("Kafka config.retry contains an unknown field");
+      for (const key of retryKeys.filter((key2) => key2 !== "restartOnFailure")) {
+        const value = config.retry[key];
+        if (value !== undefined && (typeof value !== "number" || !Number.isFinite(value) || value < 0))
+          throw new Error(`Kafka config.retry.${key} must be a nonnegative number`);
+      }
+      if (config.retry.restartOnFailure !== undefined && typeof config.retry.restartOnFailure !== "function")
+        throw new Error("Kafka config.retry.restartOnFailure must be a function");
+    }
+    if (config.sasl !== undefined) {
+      const sasl = config.sasl;
+      if (!object(sasl) || typeof sasl.mechanism !== "string" || !sasl.mechanism.trim())
+        throw new Error("Kafka config.sasl.mechanism must be a nonempty string");
+      if (sasl.authenticationProvider !== undefined) {
+        if (typeof sasl.authenticationProvider !== "function")
+          throw new Error("Kafka config.sasl.authenticationProvider must be a function");
+      } else if (["plain", "scram-sha-256", "scram-sha-512"].includes(sasl.mechanism)) {
+        for (const key of ["username", "password"]) {
+          if (typeof sasl[key] !== "string" || !sasl[key].trim())
+            throw new Error(`Kafka config.sasl.${key} must be a nonempty string`);
+        }
+      } else if (sasl.mechanism === "oauthbearer") {
+        if (typeof sasl.oauthBearerProvider !== "function")
+          throw new Error("Kafka config.sasl.oauthBearerProvider must be a function");
+      } else if (sasl.mechanism === "aws") {
+        for (const key of ["authorizationIdentity", "accessKeyId", "secretAccessKey"]) {
+          if (typeof sasl[key] !== "string" || !sasl[key].trim())
+            throw new Error(`Kafka config.sasl.${key} must be a nonempty string`);
+        }
+        if (sasl.sessionToken !== undefined && typeof sasl.sessionToken !== "string")
+          throw new Error("Kafka config.sasl.sessionToken must be a string");
+      } else
+        throw new Error("Kafka config.sasl requires a supported mechanism or authenticationProvider");
+    }
+  },
+  create(options) {
+    return new ServeKafka(options.config, options.consumerPath, options.producerPath, { rootDir: options.rootDir });
+  }
+};
+var plugin_default = definition;
+export {
+  plugin_default as default
+};

package/package.json

@@ -1,6 +1,37 @@
 {
   "name": "@beespoke/micro-server-kafka",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "2.0.3",
+  "type": "module",
+  "license": "MIT",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./plugin": {
+      "types": "./dist/plugin/index.d.ts",
+      "import": "./dist/plugin/index.js",
+      "default": "./dist/plugin/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "bun ../../scripts/build-package.ts"
+  },
+  "dependencies": {
+    "kafkajs": "^2.2.4",
+    "@types/node": "^24.0.0"
+  },
+  "peerDependencies": {
+    "@beespoke/micro-server": "^2.0.3"
+  },
+  "peerDependenciesMeta": {
+    "@beespoke/micro-server": {
+      "optional": true
+    }
+  }
+}