@dbos-inc/confluent-kafka-receive 3.0.16-preview

package/README.md

@@ -0,0 +1,152 @@
+# DBOS Confluent Kafka Receiver
+
+Publish/subscribe message queues are a common building block for distributed systems.
+Message queues allow processing to occur at a different place or time, perhaps in multiple client programming environments.
+Due to its performance, flexibility, and simple, scalable design, [Kafka](https://www.confluent.io/cloud-kafka) is a popular choice for publish/subscribe.
+
+This package includes a [DBOS](https://docs.dbos.dev/) receiver for Kafka messages, which reliably invokes a
+[DBOS workflow](https://docs.dbos.dev/typescript/tutorials/workflow-tutorial) for every Kafka message received.
+
+This package is based on the [Confluent JavaScript Client for Apache Kafka](https://github.com/confluentinc/confluent-kafka-javascript/).
+
+## Configuring a Confluent Kafka Receiver
+
+First, ensure that the DBOS Confluent Kafka Receiver package is installed into the application:
+
+```
+npm install --save @dbos-inc/confluent-kafka-receive
+```
+
+Then, create a `ConfluentKafkaReceiver` instance, providing the Kafka configuration information to the constructor.
+
+```ts
+const kafkaConfig = {
+  clientId: 'example-dbos-kafka-client',
+  brokers: ['kafka-host:9092'],
+};
+
+const kafkaReceiver = new ConfluentKafkaReceiver(kafkaConfig);
+```
+
+Finally, register a DBOS workflow as a Kafka topic consumer via the `KafkaReceiver` instance.
+This can be done with the `KafkaReceiver.consumer` decorator or the `KafkaReceiver.registerConsumer` function.
+
+```ts
+class KafkaExample {
+  @kafkaReceiver.consumer('example-topic')
+  @DBOS.workflow()
+  static async consumerWorkflow(topic: string, partition: number, message: ConfluentKafkaJS.Message) {
+    DBOS.logger.info(`Message received: ${message.value}`);
+  }
+
+  static async registeredConsumerWorkflow(topic: string, partition: number, message: ConfluentKafkaJS.Message) {
+    DBOS.logger.info(`Message received: ${message.value}`);
+  }
+}
+
+KafkaExample.registeredConsumerWorkflow = DBOS.registerWorkflow(
+  KafkaExample.registeredConsumerWorkflow,
+  'registeredConsumerWorkflow',
+);
+kafkaReceiver.registerConsumer(KafkaExample.registeredConsumerWorkflow, 'another-example-topic');
+```
+
+When registering a Kafka consumer workflow, you can specify a single topic or an array of topics.
+Topics are specified as strings or regular expressions.
+
+### Kafka Consumer Configuration
+
+If you need more control, you can pass consumer configuration into the decorator or `register` function.
+Additionally, if you need managed concurrency, you can specify the [DBOS Queue](https://docs.dbos.dev/typescript/tutorials/queue-tutorial)
+to use when executing the workflow.
+
+```ts
+class KafkaExample {
+  @kafkaReceiver.consumer('example-topic', {
+    config: { groupId: 'custom-group-id' },
+  })
+  @DBOS.workflow()
+  static async consumerWorkflow(topic: string, partition: number, message: KafkaMessage) {
+    DBOS.logger.info(`Message received: ${message.value}`);
+  }
+
+  static async registeredConsumerWorkflow(topic: string, partition: number, message: KafkaMessage) {
+    DBOS.logger.info(`Message received: ${message.value}`);
+  }
+}
+
+KafkaExample.registeredConsumerWorkflow = DBOS.registerWorkflow(
+  KafkaExample.registeredConsumerWorkflow,
+  'registeredConsumerWorkflow',
+);
+kafkaReceiver.registerConsumer(KafkaExample.registeredConsumerWorkflow, 'another-example-topic', {
+  config: { groupId: 'custom-group-id' },
+});
+```
+
+### Concurrency and Rate Limiting
+
+By default, Kafka `eventConsumer` workflows are started immediately after message receipt.
+If `queueName` is specified in `eventConsumer` options, then the workflow will be enqueued in a [workflow queue](https://docs.dbos.dev/typescript/reference/transactapi/workflow-queues).
+
+```ts
+class KafkaExample {
+  @kafkaReceiver.consumer('example-topic', { queueName: 'example-queue' })
+  @DBOS.workflow()
+  static async consumerWorkflow(topic: string, partition: number, message: KafkaMessage) {
+    DBOS.logger.info(`Message received: ${message.value}`);
+  }
+
+  static async registeredConsumerWorkflow(topic: string, partition: number, message: KafkaMessage) {
+    DBOS.logger.info(`Message received: ${message.value}`);
+  }
+}
+
+KafkaExample.registeredConsumerWorkflow = DBOS.registerWorkflow(
+  KafkaExample.registeredConsumerWorkflow,
+  'registeredConsumerWorkflow',
+);
+kafkaReceiver.registerConsumer(KafkaExample.registeredConsumerWorkflow, 'another-example-topic', {
+  queueName: 'example-queue',
+});
+```
+
+## Sending Messages
+
+Sending Kafka messages is done directly using the KafkaJS library.
+You can wrap the message send call in a DBOS Step to make it reliable.
+
+```ts
+import { KafkaJS } from '@confluentinc/kafka-javascript';
+
+class KafkaTestClass {
+  @DBOS.workflow()
+  static async kafkaSendWorkflow(name: string, value: number) {
+    const kafka = new KafkaJS.Kafka({ kafkaJS: kafkaConfig });
+
+    const producer = kafka.producer();
+    await producer.connect();
+
+    try {
+      DBOS.runStep(
+        async () => {
+          const message = JSON.stringify({ name, value });
+          await producer.send({
+            topic: 'example-topic',
+            messages: [{ value: message }],
+          });
+        },
+        { name: 'send-kafka-message' },
+      );
+    } finally {
+      await producer.disconnect();
+    }
+  }
+}
+```
+
+## Next Steps
+
+- To start a DBOS app from a template, visit our [quickstart](https://docs.dbos.dev/quickstart).
+- For DBOS programming tutorials, check out our [programming guide](https://docs.dbos.dev/typescript/programming-guide).
+- To learn more about DBOS, take a look at [our documentation](https://docs.dbos.dev/) or our [source code](https://github.com/dbos-inc/dbos-transact-ts).

package/dist/index.d.ts

@@ -0,0 +1,30 @@
+import { DBOSLifecycleCallback } from '@dbos-inc/dbos-sdk';
+import { KafkaJS } from '@confluentinc/kafka-javascript';
+interface KafkaRetryConfig {
+    maxRetries: number;
+    retryTime: number;
+    multiplier: number;
+}
+export type ConsumerTopics = string | RegExp | Array<string | RegExp>;
+export declare class ConfluentKafkaReceiver extends DBOSLifecycleCallback {
+    #private;
+    private readonly config;
+    private readonly retryConfig;
+    constructor(config: KafkaJS.KafkaConfig, retryConfig?: KafkaRetryConfig);
+    initialize(): Promise<void>;
+    destroy(): Promise<void>;
+    logRegisteredEndpoints(): void;
+    registerConsumer<This, Args extends unknown[], Return>(func: (this: This, ...args: Args) => Promise<Return>, topics: ConsumerTopics, options?: {
+        classOrInst?: object;
+        className?: string;
+        name?: string;
+        queueName?: string;
+        config?: KafkaJS.ConsumerConstructorConfig;
+    }): void;
+    consumer(topics: ConsumerTopics, options?: {
+        queueName?: string;
+        config?: KafkaJS.ConsumerConstructorConfig;
+    }): <This, Args extends [string, number, KafkaJS.Message], Return>(target: object, propertyKey: PropertyKey, descriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise<Return>>) => TypedPropertyDescriptor<(this: This, ...args: Args) => Promise<Return>>;
+}
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAEjE,OAAO,EAAE,OAAO,EAAiC,MAAM,gCAAgC,CAAC;AAYxF,UAAU,gBAAgB;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAiBD,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;AAEtE,qBAAa,sBAAuB,SAAQ,qBAAqB;;IAI7D,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,WAAW;gBADX,MAAM,EAAE,OAAO,CAAC,WAAW,EAC3B,WAAW,GAAE,gBAAmE;IAMpF,UAAU;IA+DV,OAAO;IAKb,sBAAsB;IAa/B,gBAAgB,CAAC,IAAI,EAAE,IAAI,SAAS,OAAO,EAAE,EAAE,MAAM,EACnD,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,EACpD,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,MAAM,CAAC,EAAE,OAAO,CAAC,yBAAyB,CAAC;KACvC;IAcR,QAAQ,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAC,yBAAyB,CAAA;KAAO,0EAIrG,MAAM,eACD,WAAW,cACZ,wBAAwB,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,IAAI,KAAK,QAAQ,MAAM,CAAC,CAAC,oCAAxC,IAAI,WAAW,IAAI,KAAK,QAAQ,MAAM,CAAC;CAcvF"}

package/dist/index.js

@@ -0,0 +1,131 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConfluentKafkaReceiver = void 0;
+const dbos_sdk_1 = require("@dbos-inc/dbos-sdk");
+const kafka_javascript_1 = require("@confluentinc/kafka-javascript");
+const sleepms = (ms) => new Promise((r) => setTimeout(r, ms));
+function safeGroupName(className, methodName, topics) {
+    const safeGroupIdPart = [className, methodName, ...topics]
+        .map((r) => r.toString())
+        .map((r) => r.replaceAll(/[^a-zA-Z0-9\\-]/g, ''))
+        .join('-');
+    return `dbos-kafka-group-${safeGroupIdPart}`.slice(0, 255);
+}
+function isKafkaError(e) {
+    if (e && typeof e === 'object') {
+        return 'code' in e && typeof e.code === 'number';
+    }
+    return false;
+}
+class ConfluentKafkaReceiver extends dbos_sdk_1.DBOSLifecycleCallback {
+    config;
+    retryConfig;
+    #consumers = new Array();
+    constructor(config, retryConfig = { maxRetries: 5, retryTime: 300, multiplier: 2 }) {
+        super();
+        this.config = config;
+        this.retryConfig = retryConfig;
+        dbos_sdk_1.DBOS.registerLifecycleCallback(this);
+    }
+    async initialize() {
+        const { maxRetries, multiplier } = this.retryConfig;
+        const clientId = this.config.clientId ?? 'dbos-confluent-kafka-receiver';
+        const kafka = new kafka_javascript_1.KafkaJS.Kafka({ kafkaJS: { ...this.config, clientId } });
+        for (const regOp of dbos_sdk_1.DBOS.getAssociatedInfo(this)) {
+            const func = regOp.methodReg.registeredFunction;
+            if (func === undefined) {
+                continue; // TODO: Log?
+            }
+            const methodConfig = regOp.methodConfig;
+            const topics = methodConfig.topics ?? [];
+            if (topics.length === 0) {
+                continue; // TODO: Log?
+            }
+            const { name, className } = regOp.methodReg;
+            const config = methodConfig.config ?? {
+                'group.id': safeGroupName(className, name, topics),
+            };
+            const consumer = kafka.consumer({ ...config, 'auto.offset.reset': 'earliest' });
+            await consumer.connect();
+            // A temporary workaround for https://github.com/tulios/kafkajs/pull/1558 until it gets fixed
+            // If topic auto-creation is on and you try to subscribe to a nonexistent topic, KafkaJS should retry until the topic is created.
+            // However, it has a bug where it won't. Thus, we retry instead.
+            let { retryTime } = this.retryConfig;
+            for (let i = 1; i <= maxRetries; i++) {
+                try {
+                    await consumer.subscribe({ topics });
+                    break;
+                }
+                catch (e) {
+                    if (isKafkaError(e) && e.code === 3 && i < maxRetries) {
+                        await sleepms(retryTime);
+                        retryTime *= multiplier;
+                    }
+                    else {
+                        throw e;
+                    }
+                }
+            }
+            await consumer.run({
+                eachMessage: async ({ topic, partition, message }) => {
+                    dbos_sdk_1.DBOS.logger.debug(`ConfluentKafkaReceiver message on topic ${topic} partition ${partition} offset ${message.offset}`);
+                    try {
+                        const workflowID = `confluent-kafka-${topic}-${partition}-${config['group.id']}-${message.offset}`;
+                        const wfParams = { workflowID, queueName: methodConfig.queueName };
+                        await dbos_sdk_1.DBOS.startWorkflow(func, wfParams)(topic, partition, message);
+                    }
+                    catch (e) {
+                        const message = e instanceof Error ? e.message : String(e);
+                        dbos_sdk_1.DBOS.logger.error(`Error processing Kafka message ${message}`);
+                        throw e;
+                    }
+                },
+            });
+            this.#consumers.push(consumer);
+        }
+    }
+    async destroy() {
+        const disconnectPromises = this.#consumers.splice(0, this.#consumers.length).map((c) => c.disconnect());
+        await Promise.allSettled(disconnectPromises);
+    }
+    logRegisteredEndpoints() {
+        dbos_sdk_1.DBOS.logger.info('KafkaJS receiver endpoints:');
+        const regOps = dbos_sdk_1.DBOS.getAssociatedInfo(this);
+        for (const regOp of regOps) {
+            const methodConfig = regOp.methodConfig;
+            const { name, className } = regOp.methodReg;
+            for (const topic of methodConfig.topics ?? []) {
+                dbos_sdk_1.DBOS.logger.info(`    ${topic} -> ${className}.${name}`);
+            }
+        }
+    }
+    registerConsumer(func, topics, options = {}) {
+        const { regInfo } = dbos_sdk_1.DBOS.associateFunctionWithInfo(this, func, {
+            classOrInst: options.classOrInst,
+            className: options.className,
+            name: options.name ?? func.name,
+        });
+        const kafkaRegInfo = regInfo;
+        kafkaRegInfo.topics = Array.isArray(topics) ? topics : [topics];
+        kafkaRegInfo.queueName = options.queueName;
+        kafkaRegInfo.config = options.config;
+    }
+    consumer(topics, options = {}) {
+        // eslint-disable-next-line @typescript-eslint/no-this-alias
+        const $this = this;
+        function methodDecorator(target, propertyKey, descriptor) {
+            if (descriptor.value) {
+                $this.registerConsumer(descriptor.value, topics, {
+                    classOrInst: target,
+                    name: String(propertyKey),
+                    queueName: options.queueName,
+                    config: options.config,
+                });
+            }
+            return descriptor;
+        }
+        return methodDecorator;
+    }
+}
+exports.ConfluentKafkaReceiver = ConfluentKafkaReceiver;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":";;;AAAA,iDAAiE;AAEjE,qEAAwF;AAIxF,MAAM,OAAO,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AActE,SAAS,aAAa,CAAC,SAAiB,EAAE,UAAkB,EAAE,MAA8B;IAC1F,MAAM,eAAe,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;SACvD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;SAChD,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,OAAO,oBAAoB,eAAe,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,YAAY,CAAC,CAAU;IAC9B,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,MAAM,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;IACnD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAID,MAAa,sBAAuB,SAAQ,gCAAqB;IAI5C;IACA;IAJV,UAAU,GAAG,IAAI,KAAK,EAAoB,CAAC;IAEpD,YACmB,MAA2B,EAC3B,cAAgC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE;QAEjG,KAAK,EAAE,CAAC;QAHS,WAAM,GAAN,MAAM,CAAqB;QAC3B,gBAAW,GAAX,WAAW,CAAqE;QAGjG,eAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAEQ,KAAK,CAAC,UAAU;QACvB,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC;QACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,+BAA+B,CAAC;QACzE,MAAM,KAAK,GAAG,IAAI,0BAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;QAE3E,KAAK,MAAM,KAAK,IAAI,eAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,kBAA8D,CAAC;YAC5F,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,SAAS,CAAC,aAAa;YACzB,CAAC;YAED,MAAM,YAAY,GAAG,KAAK,CAAC,YAAiC,CAAC;YAC7D,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,IAAI,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,SAAS,CAAC,aAAa;YACzB,CAAC;YAED,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC;YAC5C,MAAM,MAAM,GAAsC,YAAY,CAAC,MAAM,IAAI;gBACvE,UAAU,EAAE,aAAa,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC;aACnD,CAAC;YACF,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,GAAG,MAAM,EAAE,mBAAmB,EAAE,UAAU,EAAE,CAAC,CAAC;YAChF,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;YAEzB,6FAA6F;YAC7F,iIAAiI;YACjI,gEAAgE;YAChE,IAAI,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,MAAM,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;oBACrC,MAAM;gBACR,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,IAAI,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC;wBACtD,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC;wBACzB,SAAS,IAAI,UAAU,CAAC;oBAC1B,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,CAAC;oBACV,CAAC;gBACH,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,CAAC,GAAG,CAAC;gBACjB,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE;oBACnD,eAAI,CAAC,MAAM,CAAC,KAAK,CACf,2CAA2C,KAAK,cAAc,SAAS,WAAW,OAAO,CAAC,MAAM,EAAE,CACnG,CAAC;oBACF,IAAI,CAAC;wBACH,MAAM,UAAU,GAAG,mBAAmB,KAAK,IAAI,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;wBACnG,MAAM,QAAQ,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,CAAC,SAAS,EAAE,CAAC;wBACnE,MAAM,eAAI,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;oBACtE,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;wBAC3D,eAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,OAAO,EAAE,CAAC,CAAC;wBAC/D,MAAM,CAAC,CAAC;oBACV,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAEQ,KAAK,CAAC,OAAO;QACpB,MAAM,kBAAkB,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;QACxG,MAAM,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;IAC/C,CAAC;IAEQ,sBAAsB;QAC7B,eAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAEhD,MAAM,MAAM,GAAG,eAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,KAAK,CAAC,YAAiC,CAAC;YAC7D,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC;YAC5C,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBAC9C,eAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,OAAO,SAAS,IAAI,IAAI,EAAE,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;IACH,CAAC;IAED,gBAAgB,CACd,IAAoD,EACpD,MAAsB,EACtB,UAMI,EAAE;QAEN,MAAM,EAAE,OAAO,EAAE,GAAG,eAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,IAAI,EAAE;YAC7D,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI;SAChC,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,OAA4B,CAAC;QAClD,YAAY,CAAC,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAChE,YAAY,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QAC3C,YAAY,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACvC,CAAC;IAED,QAAQ,CAAC,MAAsB,EAAE,UAA8E,EAAE;QAC/G,4DAA4D;QAC5D,MAAM,KAAK,GAAG,IAAI,CAAC;QACnB,SAAS,eAAe,CACtB,MAAc,EACd,WAAwB,EACxB,UAAmF;YAEnF,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,KAAK,CAAC,gBAAgB,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE;oBAC/C,WAAW,EAAE,MAAM;oBACnB,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC;oBACzB,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;iBACvB,CAAC,CAAC;YACL,CAAC;YACD,OAAO,UAAU,CAAC;QACpB,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC;CACF;AAvID,wDAuIC"}

package/index.ts

@@ -0,0 +1,173 @@
+import { DBOS, DBOSLifecycleCallback } from '@dbos-inc/dbos-sdk';
+
+import { KafkaJS, LibrdKafkaError as KafkaError } from '@confluentinc/kafka-javascript';
+
+type KafkaMessageHandler<Return> = (topic: string, partition: number, message: KafkaJS.Message) => Promise<Return>;
+
+const sleepms = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+interface KafkaMethodConfig {
+  topics?: Array<string | RegExp>;
+  config?: KafkaJS.ConsumerConstructorConfig;
+  queueName?: string;
+}
+
+interface KafkaRetryConfig {
+  maxRetries: number;
+  retryTime: number;
+  multiplier: number;
+}
+
+function safeGroupName(className: string, methodName: string, topics: Array<string | RegExp>) {
+  const safeGroupIdPart = [className, methodName, ...topics]
+    .map((r) => r.toString())
+    .map((r) => r.replaceAll(/[^a-zA-Z0-9\\-]/g, ''))
+    .join('-');
+  return `dbos-kafka-group-${safeGroupIdPart}`.slice(0, 255);
+}
+
+function isKafkaError(e: unknown): e is KafkaError {
+  if (e && typeof e === 'object') {
+    return 'code' in e && typeof e.code === 'number';
+  }
+  return false;
+}
+
+export type ConsumerTopics = string | RegExp | Array<string | RegExp>;
+
+export class ConfluentKafkaReceiver extends DBOSLifecycleCallback {
+  readonly #consumers = new Array<KafkaJS.Consumer>();
+
+  constructor(
+    private readonly config: KafkaJS.KafkaConfig,
+    private readonly retryConfig: KafkaRetryConfig = { maxRetries: 5, retryTime: 300, multiplier: 2 },
+  ) {
+    super();
+    DBOS.registerLifecycleCallback(this);
+  }
+
+  override async initialize() {
+    const { maxRetries, multiplier } = this.retryConfig;
+    const clientId = this.config.clientId ?? 'dbos-confluent-kafka-receiver';
+    const kafka = new KafkaJS.Kafka({ kafkaJS: { ...this.config, clientId } });
+
+    for (const regOp of DBOS.getAssociatedInfo(this)) {
+      const func = regOp.methodReg.registeredFunction as KafkaMessageHandler<unknown> | undefined;
+      if (func === undefined) {
+        continue; // TODO: Log?
+      }
+
+      const methodConfig = regOp.methodConfig as KafkaMethodConfig;
+      const topics = methodConfig.topics ?? [];
+      if (topics.length === 0) {
+        continue; // TODO: Log?
+      }
+
+      const { name, className } = regOp.methodReg;
+      const config: KafkaJS.ConsumerConstructorConfig = methodConfig.config ?? {
+        'group.id': safeGroupName(className, name, topics),
+      };
+      const consumer = kafka.consumer({ ...config, 'auto.offset.reset': 'earliest' });
+      await consumer.connect();
+
+      // A temporary workaround for https://github.com/tulios/kafkajs/pull/1558 until it gets fixed
+      // If topic auto-creation is on and you try to subscribe to a nonexistent topic, KafkaJS should retry until the topic is created.
+      // However, it has a bug where it won't. Thus, we retry instead.
+      let { retryTime } = this.retryConfig;
+      for (let i = 1; i <= maxRetries; i++) {
+        try {
+          await consumer.subscribe({ topics });
+          break;
+        } catch (e) {
+          if (isKafkaError(e) && e.code === 3 && i < maxRetries) {
+            await sleepms(retryTime);
+            retryTime *= multiplier;
+          } else {
+            throw e;
+          }
+        }
+      }
+
+      await consumer.run({
+        eachMessage: async ({ topic, partition, message }) => {
+          DBOS.logger.debug(
+            `ConfluentKafkaReceiver message on topic ${topic} partition ${partition} offset ${message.offset}`,
+          );
+          try {
+            const workflowID = `confluent-kafka-${topic}-${partition}-${config['group.id']}-${message.offset}`;
+            const wfParams = { workflowID, queueName: methodConfig.queueName };
+            await DBOS.startWorkflow(func, wfParams)(topic, partition, message);
+          } catch (e) {
+            const message = e instanceof Error ? e.message : String(e);
+            DBOS.logger.error(`Error processing Kafka message ${message}`);
+            throw e;
+          }
+        },
+      });
+
+      this.#consumers.push(consumer);
+    }
+  }
+
+  override async destroy() {
+    const disconnectPromises = this.#consumers.splice(0, this.#consumers.length).map((c) => c.disconnect());
+    await Promise.allSettled(disconnectPromises);
+  }
+
+  override logRegisteredEndpoints() {
+    DBOS.logger.info('KafkaJS receiver endpoints:');
+
+    const regOps = DBOS.getAssociatedInfo(this);
+    for (const regOp of regOps) {
+      const methodConfig = regOp.methodConfig as KafkaMethodConfig;
+      const { name, className } = regOp.methodReg;
+      for (const topic of methodConfig.topics ?? []) {
+        DBOS.logger.info(`    ${topic} -> ${className}.${name}`);
+      }
+    }
+  }
+
+  registerConsumer<This, Args extends unknown[], Return>(
+    func: (this: This, ...args: Args) => Promise<Return>,
+    topics: ConsumerTopics,
+    options: {
+      classOrInst?: object;
+      className?: string;
+      name?: string;
+      queueName?: string;
+      config?: KafkaJS.ConsumerConstructorConfig;
+    } = {},
+  ) {
+    const { regInfo } = DBOS.associateFunctionWithInfo(this, func, {
+      classOrInst: options.classOrInst,
+      className: options.className,
+      name: options.name ?? func.name,
+    });
+
+    const kafkaRegInfo = regInfo as KafkaMethodConfig;
+    kafkaRegInfo.topics = Array.isArray(topics) ? topics : [topics];
+    kafkaRegInfo.queueName = options.queueName;
+    kafkaRegInfo.config = options.config;
+  }
+
+  consumer(topics: ConsumerTopics, options: { queueName?: string; config?: KafkaJS.ConsumerConstructorConfig } = {}) {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const $this = this;
+    function methodDecorator<This, Args extends [string, number, KafkaJS.Message], Return>(
+      target: object,
+      propertyKey: PropertyKey,
+      descriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise<Return>>,
+    ) {
+      if (descriptor.value) {
+        $this.registerConsumer(descriptor.value, topics, {
+          classOrInst: target,
+          name: String(propertyKey),
+          queueName: options.queueName,
+          config: options.config,
+        });
+      }
+      return descriptor;
+    }
+    return methodDecorator;
+  }
+}

package/jest.config.js

@@ -0,0 +1,8 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testRegex: '((\\.|/)(test|spec))\\.ts?$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePaths: ['./'],
+};

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@dbos-inc/confluent-kafka-receive",
+  "version": "3.0.16-preview",
+  "description": "DBOS event reciever for Kafka using the Confluent JavaScript Client for Apache Kafka",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "homepage": "https://docs.dbos.dev/",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbos-inc/dbos-transact-ts",
+    "directory": "packages/confluent-kafka-receive"
+  },
+  "scripts": {
+    "build": "tsc --project tsconfig.build.json",
+    "pretest": "npm run build",
+    "test": "tsx --test tests/*.test.ts"
+  },
+  "dependencies": {
+    "@confluentinc/kafka-javascript": "^1.3.2"
+  },
+  "peerDependencies": {
+    "@dbos-inc/dbos-sdk": "*"
+  },
+  "devDependencies": {
+    "kafkajs": "^2.2.4",
+    "tsx": "^4.20.3"
+  }
+}

package/tests/confluent-kafka.test.ts

@@ -0,0 +1,248 @@
+import { after, afterEach, before, beforeEach, suite, test } from 'node:test';
+import assert from 'node:assert/strict';
+
+import { DBOS } from '@dbos-inc/dbos-sdk';
+import { Client } from 'pg';
+import { dropDB, withTimeout } from './test-helpers';
+import { Kafka, KafkaConfig, Producer, Admin, ConfigResourceTypes } from 'kafkajs';
+import { EventEmitter } from 'node:events';
+import { ConfluentKafkaReceiver } from '..';
+import { KafkaJS as ConfluentKafkaJS } from '@confluentinc/kafka-javascript';
+
+const kafkaConfig = {
+  clientId: 'dbos-conf-kafka-test',
+  brokers: [process.env['KAFKA_BROKER'] ?? 'localhost:9092'],
+  retry: { retries: 5 },
+  logLevel: 2,
+};
+
+const kafkaReceiver = new ConfluentKafkaReceiver(kafkaConfig);
+
+interface KafkaEvents {
+  message: (functionName: string, topic: string, partition: number, message: ConfluentKafkaJS.Message) => void;
+}
+
+class KafkaEmitter extends EventEmitter {
+  override on<K extends keyof KafkaEvents>(event: K, listener: KafkaEvents[K]): this {
+    return super.on(event, listener);
+  }
+
+  override emit<K extends keyof KafkaEvents>(event: K, ...args: Parameters<KafkaEvents[K]>): boolean {
+    DBOS.logger.info(`KafkaEmitter topic ${args[1]} partition ${args[2]}`);
+    return super.emit(event, ...args);
+  }
+}
+
+type KafkaMessageEvent = {
+  topic: string;
+  partition: number;
+  message: ConfluentKafkaJS.Message;
+};
+
+function waitForMessage(
+  emitter: KafkaEmitter,
+  funcName: string,
+  topic: string,
+  timeoutMS = 45000,
+): Promise<KafkaMessageEvent> {
+  return withTimeout(
+    new Promise<KafkaMessageEvent>((resolve) => {
+      const handler = (f: string, t: string, partition: number, message: ConfluentKafkaJS.Message) => {
+        if (f === funcName && t === topic) {
+          emitter.off('message', handler);
+          resolve({ topic: t, partition, message });
+        }
+      };
+      emitter.on('message', handler);
+    }),
+    timeoutMS,
+    `Timeout waiting for message for function ${funcName}`,
+  );
+}
+
+class KafkaTestClass {
+  static readonly emitter = new KafkaEmitter();
+
+  @kafkaReceiver.consumer('string-topic')
+  @DBOS.workflow()
+  static async stringTopic(topic: string, partition: number, message: ConfluentKafkaJS.Message) {
+    await Promise.resolve();
+    KafkaTestClass.emitter.emit('message', 'stringTopic', topic, partition, message);
+  }
+
+  @kafkaReceiver.consumer(/^regex-topic-.*/)
+  @DBOS.workflow()
+  static async regexTopic(topic: string, partition: number, message: ConfluentKafkaJS.Message) {
+    await Promise.resolve();
+    KafkaTestClass.emitter.emit('message', 'regexTopic', topic, partition, message);
+  }
+
+  @kafkaReceiver.consumer(['a-topic', 'b-topic'])
+  @DBOS.workflow()
+  static async stringArrayTopic(topic: string, partition: number, message: ConfluentKafkaJS.Message) {
+    await Promise.resolve();
+    KafkaTestClass.emitter.emit('message', 'stringArrayTopic', topic, partition, message);
+  }
+
+  @kafkaReceiver.consumer([/^z-topic-.*/, /^y-topic-.*/])
+  @DBOS.workflow()
+  static async regexArrayTopic(topic: string, partition: number, message: ConfluentKafkaJS.Message) {
+    await Promise.resolve();
+    KafkaTestClass.emitter.emit('message', 'regexArrayTopic', topic, partition, message);
+  }
+
+  static async registeredConsumer(topic: string, partition: number, message: ConfluentKafkaJS.Message) {
+    await Promise.resolve();
+    KafkaTestClass.emitter.emit('message', 'registeredConsumer', topic, partition, message);
+  }
+}
+
+KafkaTestClass.registeredConsumer = DBOS.registerWorkflow(KafkaTestClass.registeredConsumer, 'registeredConsumer');
+kafkaReceiver.registerConsumer(KafkaTestClass.registeredConsumer, 'registered-topic');
+
+async function validateKafka(config: KafkaConfig) {
+  const kafka = new Kafka(config);
+  const admin = kafka.admin();
+  try {
+    await admin.connect();
+    await admin.listTopics();
+    return true;
+  } catch (e) {
+    const message = e instanceof Error ? e.message : String(e);
+    DBOS.logger.error(message);
+    return false;
+  } finally {
+    await admin.disconnect();
+  }
+}
+
+async function createTopics(admin: Admin, topics: string[]) {
+  const existingTopics = await admin.listTopics();
+  const topicsToCreate = topics.filter((t) => !existingTopics.includes(t));
+  await admin.createTopics({
+    topics: topicsToCreate.map((t) => ({
+      topic: t,
+      numPartitions: 1,
+      replicationFactor: 1,
+    })),
+    timeout: 5000,
+  });
+}
+
+async function purgeTopic(admin: Admin, topic: string) {
+  const { resources } = await admin.describeConfigs({
+    includeSynonyms: false,
+    resources: [
+      {
+        type: ConfigResourceTypes.TOPIC,
+        name: topic,
+        configNames: ['retention.ms'],
+      },
+    ],
+  });
+
+  const resource = resources.find((r) => r.resourceName === topic);
+  const configEntry = (resource?.configEntries ?? []).find((ce) => ce.configName === 'retention.ms');
+  const retentionMS = configEntry?.configValue ?? '604800000';
+
+  await admin.alterConfigs({
+    validateOnly: false,
+    resources: [
+      {
+        type: ConfigResourceTypes.TOPIC,
+        name: topic,
+        configEntries: [{ name: 'retention.ms', value: '0' }],
+      },
+    ],
+  });
+
+  await new Promise((resolve) => setTimeout(resolve, 2000));
+
+  await admin.alterConfigs({
+    validateOnly: false,
+    resources: [
+      {
+        type: ConfigResourceTypes.TOPIC,
+        name: topic,
+        configEntries: [{ name: 'retention.ms', value: retentionMS }],
+      },
+    ],
+  });
+
+  await new Promise((resolve) => setTimeout(resolve, 2000));
+}
+
+// eslint-disable-next-line @typescript-eslint/no-floating-promises
+suite('confluent-kafka-receive', async () => {
+  const kafkaAvailable = await validateKafka(kafkaConfig);
+  let producer: Producer | undefined = undefined;
+  let admin: Admin | undefined = undefined;
+
+  const testCases = [
+    { topic: 'string-topic', functionName: 'stringTopic' },
+    { topic: 'regex-topic-foo', functionName: 'regexTopic' },
+    { topic: 'a-topic', functionName: 'stringArrayTopic' },
+    { topic: 'b-topic', functionName: 'stringArrayTopic' },
+    { topic: 'z-topic-foo', functionName: 'regexArrayTopic' },
+    { topic: 'y-topic-foo', functionName: 'regexArrayTopic' },
+    { topic: 'registered-topic', functionName: 'registeredConsumer' },
+  ];
+
+  before(async () => {
+    if (!kafkaAvailable) {
+      return;
+    }
+
+    const kafka = new Kafka(kafkaConfig);
+    producer = kafka.producer();
+    admin = kafka.admin();
+    await Promise.all([producer.connect(), admin.connect()]);
+
+    await createTopics(
+      admin,
+      testCases.map((tc) => tc.topic),
+    );
+
+    const client = new Client({ user: 'postgres', database: 'postgres' });
+    try {
+      await client.connect();
+      await Promise.all([
+        dropDB(client, 'conf_kafka_recv_test', true),
+        dropDB(client, 'conf_kafka_recv_test_dbos_sys', true),
+      ]);
+    } finally {
+      await client.end();
+    }
+  });
+
+  after(async () => {
+    await admin?.disconnect();
+    await producer?.disconnect();
+  });
+
+  beforeEach(async () => {
+    if (kafkaAvailable) {
+      DBOS.setConfig({ name: 'conf-kafka-recv-test' });
+      await DBOS.launch();
+    }
+  });
+
+  afterEach(async () => {
+    if (kafkaAvailable) {
+      await DBOS.shutdown();
+    }
+  });
+
+  for (const { functionName, topic } of testCases) {
+    // eslint-disable-next-line @typescript-eslint/no-floating-promises
+    test(`${topic}-${functionName}`, { skip: !kafkaAvailable, timeout: 125000 }, async () => {
+      await purgeTopic(admin!, topic);
+      const message = `test-message-${Date.now()}`;
+      await producer!.send({ topic, messages: [{ value: message }] });
+      const result = await waitForMessage(KafkaTestClass.emitter, functionName, topic, 120000);
+
+      assert.equal(topic, result.topic);
+      assert.equal(message, String(result.message.value));
+    });
+  }
+});

package/tests/test-helpers.ts

@@ -0,0 +1,23 @@
+import { Client } from 'pg';
+
+export async function ensureDB(client: Client, name: string) {
+  const results = await client.query('SELECT 1 FROM pg_database WHERE datname = $1', [name]);
+  if (results.rows.length === 0) {
+    await client.query(`CREATE DATABASE ${name}`);
+  }
+}
+
+export async function dropDB(client: Client, name: string, force: boolean = false) {
+  const withForce = force ? ' WITH (FORCE)' : '';
+  await client.query(`DROP DATABASE IF EXISTS ${name} ${withForce}`);
+}
+
+export function withTimeout<T>(promise: Promise<T>, ms: number, message = 'Timeout'): Promise<T> {
+  let timeoutId: ReturnType<typeof setTimeout>;
+
+  const timeout = new Promise<never>((_, reject) => {
+    timeoutId = setTimeout(() => reject(new Error(message)), ms);
+  });
+
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(timeoutId));
+}

package/tsconfig.build.json

@@ -0,0 +1,8 @@
+/* Visit https://aka.ms/tsconfig to read more about this file */
+{
+  "extends": "../../tsconfig.shared.json",
+  "compilerOptions": {
+    "outDir": "./dist"
+  },
+  "include": ["index.ts"]
+}

package/tsconfig.json

@@ -0,0 +1,8 @@
+/* Visit https://aka.ms/tsconfig to read more about this file */
+{
+  "extends": "../../tsconfig.shared.json",
+  "compilerOptions": {
+    "noEmit": true
+  },
+  "include": ["index.ts", "tests/*.ts"]
+}