@forklaunch/implementation-worker-kafka 0.8.6 → 0.9.0

package/lib/consumers/index.d.mts

@@ -1,3 +1,4 @@
+import { OpenTelemetryCollector, MetricsDefinition } from '@forklaunch/core/http';
 import { WorkerConsumer } from '@forklaunch/interfaces-worker/interfaces';
 import { WorkerEventEntity, WorkerProcessFunction, WorkerFailureHandler } from '@forklaunch/interfaces-worker/types';
 import { KafkaWorkerOptions } from '../domain/types/index.mjs';
@@ -11,7 +12,8 @@ declare class KafkaWorkerConsumer<EventEntity extends WorkerEventEntity, Options
     protected readonly options: Options;
     protected readonly processEventsFunction: WorkerProcessFunction<EventEntity>;
     protected readonly failureHandler: WorkerFailureHandler<EventEntity>;
-    constructor(queueName: string, options: Options, processEventsFunction: WorkerProcessFunction<EventEntity>, failureHandler: WorkerFailureHandler<EventEntity>);
+    protected readonly openTelemetryCollector: OpenTelemetryCollector<MetricsDefinition>;
+    constructor(queueName: string, options: Options, processEventsFunction: WorkerProcessFunction<EventEntity>, failureHandler: WorkerFailureHandler<EventEntity>, openTelemetryCollector: OpenTelemetryCollector<MetricsDefinition>);
     private setupConsumer;
     peekEvents(): Promise<EventEntity[]>;
     start(): Promise<void>;

package/lib/consumers/index.d.ts

@@ -1,3 +1,4 @@
+import { OpenTelemetryCollector, MetricsDefinition } from '@forklaunch/core/http';
 import { WorkerConsumer } from '@forklaunch/interfaces-worker/interfaces';
 import { WorkerEventEntity, WorkerProcessFunction, WorkerFailureHandler } from '@forklaunch/interfaces-worker/types';
 import { KafkaWorkerOptions } from '../domain/types/index.js';
@@ -11,7 +12,8 @@ declare class KafkaWorkerConsumer<EventEntity extends WorkerEventEntity, Options
     protected readonly options: Options;
     protected readonly processEventsFunction: WorkerProcessFunction<EventEntity>;
     protected readonly failureHandler: WorkerFailureHandler<EventEntity>;
-    constructor(queueName: string, options: Options, processEventsFunction: WorkerProcessFunction<EventEntity>, failureHandler: WorkerFailureHandler<EventEntity>);
+    protected readonly openTelemetryCollector: OpenTelemetryCollector<MetricsDefinition>;
+    constructor(queueName: string, options: Options, processEventsFunction: WorkerProcessFunction<EventEntity>, failureHandler: WorkerFailureHandler<EventEntity>, openTelemetryCollector: OpenTelemetryCollector<MetricsDefinition>);
     private setupConsumer;
     peekEvents(): Promise<EventEntity[]>;
     start(): Promise<void>;

package/lib/consumers/index.js

@@ -35,7 +35,8 @@ var KafkaWorkerConsumer = class {
   options;
   processEventsFunction;
   failureHandler;
-  constructor(queueName, options, processEventsFunction, failureHandler) {
+  openTelemetryCollector;
+  constructor(queueName, options, processEventsFunction, failureHandler, openTelemetryCollector) {
     this.queueName = queueName;
     this.options = options;
     this.processEventsFunction = processEventsFunction;
@@ -48,6 +49,7 @@ var KafkaWorkerConsumer = class {
     this.consumer = this.kafka.consumer({
       groupId: this.options.groupId
     });
+    this.openTelemetryCollector = openTelemetryCollector;
   }
   async setupConsumer() {
     await this.consumer.connect();
@@ -163,8 +165,29 @@ var KafkaWorkerConsumer = class {
     }
   }
   async start() {
-    await this.setupConsumer();
-    await this.producer.connect();
+    const maxAttempts = 30;
+    const delayMs = 2e3;
+    let attempt = 1;
+    while (attempt <= maxAttempts) {
+      try {
+        await this.setupConsumer();
+        await this.producer.connect();
+        return;
+      } catch (error) {
+        const err = error;
+        const isUnknownTopic = err?.code === 3 || err?.type === "UNKNOWN_TOPIC_OR_PARTITION" || (err?.message || "").includes(
+          "This server does not host this topic-partition"
+        );
+        if (!isUnknownTopic || attempt >= maxAttempts) {
+          throw error;
+        }
+        this.openTelemetryCollector.warn(
+          `Kafka not ready for topic ${this.queueName} (attempt ${attempt}/${maxAttempts}). Retrying in ${delayMs}ms...`
+        );
+        await new Promise((r) => setTimeout(r, delayMs));
+        attempt += 1;
+      }
+    }
   }
   async close() {
     await this.producer.disconnect();

package/lib/consumers/index.mjs

@@ -9,7 +9,8 @@ var KafkaWorkerConsumer = class {
   options;
   processEventsFunction;
   failureHandler;
-  constructor(queueName, options, processEventsFunction, failureHandler) {
+  openTelemetryCollector;
+  constructor(queueName, options, processEventsFunction, failureHandler, openTelemetryCollector) {
     this.queueName = queueName;
     this.options = options;
     this.processEventsFunction = processEventsFunction;
@@ -22,6 +23,7 @@ var KafkaWorkerConsumer = class {
     this.consumer = this.kafka.consumer({
       groupId: this.options.groupId
     });
+    this.openTelemetryCollector = openTelemetryCollector;
   }
   async setupConsumer() {
     await this.consumer.connect();
@@ -137,8 +139,29 @@ var KafkaWorkerConsumer = class {
     }
   }
   async start() {
-    await this.setupConsumer();
-    await this.producer.connect();
+    const maxAttempts = 30;
+    const delayMs = 2e3;
+    let attempt = 1;
+    while (attempt <= maxAttempts) {
+      try {
+        await this.setupConsumer();
+        await this.producer.connect();
+        return;
+      } catch (error) {
+        const err = error;
+        const isUnknownTopic = err?.code === 3 || err?.type === "UNKNOWN_TOPIC_OR_PARTITION" || (err?.message || "").includes(
+          "This server does not host this topic-partition"
+        );
+        if (!isUnknownTopic || attempt >= maxAttempts) {
+          throw error;
+        }
+        this.openTelemetryCollector.warn(
+          `Kafka not ready for topic ${this.queueName} (attempt ${attempt}/${maxAttempts}). Retrying in ${delayMs}ms...`
+        );
+        await new Promise((r) => setTimeout(r, delayMs));
+        attempt += 1;
+      }
+    }
   }
   async close() {
     await this.producer.disconnect();

package/lib/eject/consumers/kafkaWorker.consumer.ts

@@ -1,3 +1,7 @@
+import {
+  MetricsDefinition,
+  OpenTelemetryCollector
+} from '@forklaunch/core/http';
 import { WorkerConsumer } from '@forklaunch/interfaces-worker/interfaces';
 import {
   WorkerEventEntity,
@@ -20,12 +24,14 @@ export class KafkaWorkerConsumer<
   protected readonly options: Options;
   protected readonly processEventsFunction: WorkerProcessFunction<EventEntity>;
   protected readonly failureHandler: WorkerFailureHandler<EventEntity>;
+  protected readonly openTelemetryCollector: OpenTelemetryCollector<MetricsDefinition>;
 
   constructor(
     queueName: string,
     options: Options,
     processEventsFunction: WorkerProcessFunction<EventEntity>,
-    failureHandler: WorkerFailureHandler<EventEntity>
+    failureHandler: WorkerFailureHandler<EventEntity>,
+    openTelemetryCollector: OpenTelemetryCollector<MetricsDefinition>
   ) {
     this.queueName = queueName;
     this.options = options;
@@ -40,6 +46,7 @@ export class KafkaWorkerConsumer<
     this.consumer = this.kafka.consumer({
       groupId: this.options.groupId
     });
+    this.openTelemetryCollector = openTelemetryCollector;
   }
 
   private async setupConsumer() {
@@ -179,8 +186,39 @@ export class KafkaWorkerConsumer<
   }
 
   async start(): Promise<void> {
-    await this.setupConsumer();
-    await this.producer.connect();
+    const maxAttempts = 30;
+    const delayMs = 2000;
+    let attempt = 1;
+
+    while (attempt <= maxAttempts) {
+      try {
+        await this.setupConsumer();
+        await this.producer.connect();
+        return;
+      } catch (error) {
+        const err = error as Error & {
+          code?: number;
+          type?: string;
+          message?: string;
+        };
+        const isUnknownTopic =
+          err?.code === 3 ||
+          err?.type === 'UNKNOWN_TOPIC_OR_PARTITION' ||
+          (err?.message || '').includes(
+            'This server does not host this topic-partition'
+          );
+
+        if (!isUnknownTopic || attempt >= maxAttempts) {
+          throw error;
+        }
+
+        this.openTelemetryCollector.warn(
+          `Kafka not ready for topic ${this.queueName} (attempt ${attempt}/${maxAttempts}). Retrying in ${delayMs}ms...`
+        );
+        await new Promise((r) => setTimeout(r, delayMs));
+        attempt += 1;
+      }
+    }
   }
 
   async close(): Promise<void> {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forklaunch/implementation-worker-kafka",
-  "version": "0.8.6",
+  "version": "0.9.0",
   "description": "Kafka implementation for forklaunch workers",
   "homepage": "https://github.com/forklaunch/forklaunch-js#readme",
   "bugs": {