@forklaunch/implementation-worker-kafka 0.2.3 → 0.3.0

package/lib/consumers/index.d.mts

@@ -0,0 +1,21 @@
+import { WorkerConsumer } from '@forklaunch/interfaces-worker/interfaces';
+import { WorkerEventEntity, WorkerProcessFunction, WorkerFailureHandler } from '@forklaunch/interfaces-worker/types';
+import { KafkaWorkerOptions } from '../types/index.mjs';
+
+declare class KafkaWorkerConsumer<EventEntity extends WorkerEventEntity, Options extends KafkaWorkerOptions> implements WorkerConsumer<EventEntity> {
+    protected readonly queueName: string;
+    protected readonly options: Options;
+    protected readonly processEventsFunction: WorkerProcessFunction<EventEntity>;
+    protected readonly failureHandler: WorkerFailureHandler<EventEntity>;
+    private kafka;
+    private producer;
+    private consumer;
+    private processedMessages;
+    constructor(queueName: string, options: Options, processEventsFunction: WorkerProcessFunction<EventEntity>, failureHandler: WorkerFailureHandler<EventEntity>);
+    private setupConsumer;
+    peekEvents(): Promise<EventEntity[]>;
+    start(): Promise<void>;
+    close(): Promise<void>;
+}
+
+export { KafkaWorkerConsumer };

package/lib/consumers/index.d.ts

@@ -1,2 +1,21 @@
-export * from './kafkaWorker.consumer';
-//# sourceMappingURL=index.d.ts.map
+import { WorkerConsumer } from '@forklaunch/interfaces-worker/interfaces';
+import { WorkerEventEntity, WorkerProcessFunction, WorkerFailureHandler } from '@forklaunch/interfaces-worker/types';
+import { KafkaWorkerOptions } from '../types/index.js';
+
+declare class KafkaWorkerConsumer<EventEntity extends WorkerEventEntity, Options extends KafkaWorkerOptions> implements WorkerConsumer<EventEntity> {
+    protected readonly queueName: string;
+    protected readonly options: Options;
+    protected readonly processEventsFunction: WorkerProcessFunction<EventEntity>;
+    protected readonly failureHandler: WorkerFailureHandler<EventEntity>;
+    private kafka;
+    private producer;
+    private consumer;
+    private processedMessages;
+    constructor(queueName: string, options: Options, processEventsFunction: WorkerProcessFunction<EventEntity>, failureHandler: WorkerFailureHandler<EventEntity>);
+    private setupConsumer;
+    peekEvents(): Promise<EventEntity[]>;
+    start(): Promise<void>;
+    close(): Promise<void>;
+}
+
+export { KafkaWorkerConsumer };

package/lib/consumers/index.js

@@ -1 +1,173 @@
-export * from './kafkaWorker.consumer';
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// consumers/index.ts
+var consumers_exports = {};
+__export(consumers_exports, {
+  KafkaWorkerConsumer: () => KafkaWorkerConsumer
+});
+module.exports = __toCommonJS(consumers_exports);
+
+// consumers/kafkaWorker.consumer.ts
+var import_kafkajs = require("kafkajs");
+var KafkaWorkerConsumer = class {
+  constructor(queueName, options, processEventsFunction, failureHandler) {
+    this.queueName = queueName;
+    this.options = options;
+    this.processEventsFunction = processEventsFunction;
+    this.failureHandler = failureHandler;
+    this.kafka = new import_kafkajs.Kafka({
+      clientId: this.options.clientId,
+      brokers: this.options.brokers
+    });
+    this.producer = this.kafka.producer();
+    this.consumer = this.kafka.consumer({
+      groupId: this.options.groupId
+    });
+  }
+  kafka;
+  producer;
+  consumer;
+  processedMessages = /* @__PURE__ */ new Set();
+  async setupConsumer() {
+    await this.consumer.connect();
+    await this.consumer.subscribe({
+      topic: this.queueName,
+      fromBeginning: false
+    });
+    await this.consumer.run({
+      eachMessage: async ({ topic, partition, message }) => {
+        if (!message.value) return;
+        const messageKey = `${topic}-${partition}-${message.offset}`;
+        if (this.processedMessages.has(messageKey)) {
+          return;
+        }
+        const events = JSON.parse(message.value.toString());
+        try {
+          await this.processEventsFunction(events);
+          this.processedMessages.add(messageKey);
+          await this.consumer.commitOffsets([
+            {
+              topic,
+              partition,
+              offset: (parseInt(message.offset) + 1).toString()
+            }
+          ]);
+        } catch (error) {
+          this.failureHandler([
+            {
+              value: events[0],
+              error
+            }
+          ]);
+          for (const event of events) {
+            if (event.retryCount <= this.options.retries) {
+              await this.producer.send({
+                topic: this.queueName,
+                messages: [
+                  {
+                    value: JSON.stringify([
+                      {
+                        ...event,
+                        retryCount: event.retryCount + 1
+                      }
+                    ]),
+                    key: event.id
+                  }
+                ]
+              });
+            }
+          }
+        }
+      }
+    });
+  }
+  async peekEvents() {
+    const events = [];
+    const admin = this.kafka.admin();
+    await admin.connect();
+    try {
+      const metadata = await admin.fetchTopicMetadata({
+        topics: [this.queueName]
+      });
+      const topic = metadata.topics[0];
+      if (!topic) {
+        return events;
+      }
+      for (const partition of topic.partitions) {
+        const offsets = await admin.fetchTopicOffsets(this.queueName);
+        const partitionOffset = offsets.find(
+          (o) => o.partition === partition.partitionId
+        );
+        if (!partitionOffset) {
+          continue;
+        }
+        const peekConsumer = this.kafka.consumer({
+          groupId: `${this.options.groupId}-peek-${Date.now()}`
+        });
+        try {
+          await peekConsumer.connect();
+          await peekConsumer.subscribe({
+            topic: this.queueName,
+            fromBeginning: false
+          });
+          const messagePromise = new Promise((resolve) => {
+            peekConsumer.run({
+              eachMessage: async ({ message }) => {
+                if (message.value && events.length < this.options.peekCount) {
+                  const messageEvents = JSON.parse(
+                    message.value.toString()
+                  );
+                  events.push(...messageEvents);
+                  if (events.length >= this.options.peekCount) {
+                    resolve();
+                  }
+                }
+              }
+            });
+          });
+          await Promise.race([
+            messagePromise,
+            new Promise((resolve) => setTimeout(resolve, 5e3))
+          ]);
+          if (events.length >= this.options.peekCount) {
+            break;
+          }
+        } finally {
+          await peekConsumer.disconnect();
+        }
+      }
+      return events;
+    } finally {
+      await admin.disconnect();
+    }
+  }
+  async start() {
+    await this.setupConsumer();
+    await this.producer.connect();
+  }
+  async close() {
+    await this.producer.disconnect();
+    await this.consumer.disconnect();
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  KafkaWorkerConsumer
+});

package/lib/consumers/{kafkaWorker.consumer.js → index.mjs}

@@ -1,13 +1,6 @@
-import { Kafka } from 'kafkajs';
-export class KafkaWorkerConsumer {
-  queueName;
-  options;
-  processEventsFunction;
-  failureHandler;
-  kafka;
-  producer;
-  consumer;
-  processedMessages = new Set();
+// consumers/kafkaWorker.consumer.ts
+import { Kafka } from "kafkajs";
+var KafkaWorkerConsumer = class {
   constructor(queueName, options, processEventsFunction, failureHandler) {
     this.queueName = queueName;
     this.options = options;
@@ -22,6 +15,10 @@ export class KafkaWorkerConsumer {
       groupId: this.options.groupId
     });
   }
+  kafka;
+  producer;
+  consumer;
+  processedMessages = /* @__PURE__ */ new Set();
   async setupConsumer() {
     await this.consumer.connect();
     await this.consumer.subscribe({
@@ -50,7 +47,7 @@ export class KafkaWorkerConsumer {
           this.failureHandler([
             {
               value: events[0],
-              error: error
+              error
             }
           ]);
           for (const event of events) {
@@ -80,7 +77,6 @@ export class KafkaWorkerConsumer {
     const admin = this.kafka.admin();
     await admin.connect();
     try {
-      // Get topic metadata to find partitions
       const metadata = await admin.fetchTopicMetadata({
         topics: [this.queueName]
       });
@@ -88,7 +84,6 @@ export class KafkaWorkerConsumer {
       if (!topic) {
         return events;
       }
-      // For each partition, get the latest offset
       for (const partition of topic.partitions) {
         const offsets = await admin.fetchTopicOffsets(this.queueName);
         const partitionOffset = offsets.find(
@@ -97,7 +92,6 @@ export class KafkaWorkerConsumer {
         if (!partitionOffset) {
           continue;
         }
-        // Create a temporary consumer to read messages
         const peekConsumer = this.kafka.consumer({
           groupId: `${this.options.groupId}-peek-${Date.now()}`
         });
@@ -111,7 +105,9 @@ export class KafkaWorkerConsumer {
             peekConsumer.run({
               eachMessage: async ({ message }) => {
                 if (message.value && events.length < this.options.peekCount) {
-                  const messageEvents = JSON.parse(message.value.toString());
+                  const messageEvents = JSON.parse(
+                    message.value.toString()
+                  );
                   events.push(...messageEvents);
                   if (events.length >= this.options.peekCount) {
                     resolve();
@@ -122,7 +118,7 @@ export class KafkaWorkerConsumer {
           });
           await Promise.race([
             messagePromise,
-            new Promise((resolve) => setTimeout(resolve, 5000))
+            new Promise((resolve) => setTimeout(resolve, 5e3))
           ]);
           if (events.length >= this.options.peekCount) {
             break;
@@ -144,4 +140,7 @@ export class KafkaWorkerConsumer {
     await this.producer.disconnect();
     await this.consumer.disconnect();
   }
-}
+};
+export {
+  KafkaWorkerConsumer
+};

package/lib/producers/index.d.mts

@@ -0,0 +1,13 @@
+import { WorkerEventEntity } from '@forklaunch/interfaces-worker/types';
+import { KafkaWorkerOptions } from '../types/index.mjs';
+
+declare class KafkaWorkerProducer<EventEntity extends WorkerEventEntity, Options extends KafkaWorkerOptions> {
+    private readonly queueName;
+    private readonly options;
+    private producer;
+    constructor(queueName: string, options: Options);
+    enqueueJob(event: EventEntity): Promise<void>;
+    enqueueBatchJobs(events: EventEntity[]): Promise<void>;
+}
+
+export { KafkaWorkerProducer };

package/lib/producers/index.d.ts

@@ -1,2 +1,13 @@
-export * from './kafkaWorker.producer';
-//# sourceMappingURL=index.d.ts.map
+import { WorkerEventEntity } from '@forklaunch/interfaces-worker/types';
+import { KafkaWorkerOptions } from '../types/index.js';
+
+declare class KafkaWorkerProducer<EventEntity extends WorkerEventEntity, Options extends KafkaWorkerOptions> {
+    private readonly queueName;
+    private readonly options;
+    private producer;
+    constructor(queueName: string, options: Options);
+    enqueueJob(event: EventEntity): Promise<void>;
+    enqueueBatchJobs(events: EventEntity[]): Promise<void>;
+}
+
+export { KafkaWorkerProducer };

package/lib/producers/index.js

@@ -1 +1,57 @@
-export * from './kafkaWorker.producer';
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// producers/index.ts
+var producers_exports = {};
+__export(producers_exports, {
+  KafkaWorkerProducer: () => KafkaWorkerProducer
+});
+module.exports = __toCommonJS(producers_exports);
+
+// producers/kafkaWorker.producer.ts
+var import_kafkajs = require("kafkajs");
+var KafkaWorkerProducer = class {
+  constructor(queueName, options) {
+    this.queueName = queueName;
+    this.options = options;
+    const kafka = new import_kafkajs.Kafka({
+      clientId: this.options.clientId,
+      brokers: this.options.brokers
+    });
+    this.producer = kafka.producer();
+    this.producer.connect();
+  }
+  producer;
+  async enqueueJob(event) {
+    await this.producer.send({
+      topic: this.queueName,
+      messages: [{ value: JSON.stringify([event]) }]
+    });
+  }
+  async enqueueBatchJobs(events) {
+    await this.producer.send({
+      topic: this.queueName,
+      messages: events.map((event) => ({ value: JSON.stringify(event) }))
+    });
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  KafkaWorkerProducer
+});

package/lib/producers/{kafkaWorker.producer.js → index.mjs}

@@ -1,8 +1,6 @@
-import { Kafka } from 'kafkajs';
-export class KafkaWorkerProducer {
-  queueName;
-  options;
-  producer;
+// producers/kafkaWorker.producer.ts
+import { Kafka } from "kafkajs";
+var KafkaWorkerProducer = class {
   constructor(queueName, options) {
     this.queueName = queueName;
     this.options = options;
@@ -13,6 +11,7 @@ export class KafkaWorkerProducer {
     this.producer = kafka.producer();
     this.producer.connect();
   }
+  producer;
   async enqueueJob(event) {
     await this.producer.send({
       topic: this.queueName,
@@ -25,4 +24,7 @@ export class KafkaWorkerProducer {
       messages: events.map((event) => ({ value: JSON.stringify(event) }))
     });
   }
-}
+};
+export {
+  KafkaWorkerProducer
+};

package/lib/schemas/index.d.mts

@@ -0,0 +1,24 @@
+import * as _forklaunch_core_mappers from '@forklaunch/core/mappers';
+import * as zod from 'zod';
+import * as _sinclair_typebox from '@sinclair/typebox';
+import * as _forklaunch_validator from '@forklaunch/validator';
+
+declare const KafkaWorkerSchemas: <SchemaValidator extends _forklaunch_validator.AnySchemaValidator>(options: Record<string, unknown> & {
+    validator: SchemaValidator;
+}) => _forklaunch_core_mappers.SchemasByValidator<SchemaValidator, (options: Record<string, unknown>) => {
+    brokers: _sinclair_typebox.TArray<_sinclair_typebox.TString>;
+    clientId: _sinclair_typebox.TString;
+    groupId: _sinclair_typebox.TString;
+    retries: _sinclair_typebox.TTransform<_sinclair_typebox.TUnion<[_sinclair_typebox.TNumber, _sinclair_typebox.TString, _sinclair_typebox.TBoolean, _sinclair_typebox.TNull, _sinclair_typebox.TBigInt, _sinclair_typebox.TDate]>, number>;
+    interval: _sinclair_typebox.TTransform<_sinclair_typebox.TUnion<[_sinclair_typebox.TNumber, _sinclair_typebox.TString, _sinclair_typebox.TBoolean, _sinclair_typebox.TNull, _sinclair_typebox.TBigInt, _sinclair_typebox.TDate]>, number>;
+    peekCount: _sinclair_typebox.TTransform<_sinclair_typebox.TUnion<[_sinclair_typebox.TNumber, _sinclair_typebox.TString, _sinclair_typebox.TBoolean, _sinclair_typebox.TNull, _sinclair_typebox.TBigInt, _sinclair_typebox.TDate]>, number>;
+}, (options: Record<string, unknown>) => {
+    brokers: zod.ZodArray<zod.ZodString, "many">;
+    clientId: zod.ZodString;
+    groupId: zod.ZodString;
+    retries: zod.ZodEffects<zod.ZodNumber, number, unknown>;
+    interval: zod.ZodEffects<zod.ZodNumber, number, unknown>;
+    peekCount: zod.ZodEffects<zod.ZodNumber, number, unknown>;
+}>;
+
+export { KafkaWorkerSchemas };

package/lib/schemas/index.d.ts

@@ -1,2 +1,24 @@
-export * from './kafka.schema';
-//# sourceMappingURL=index.d.ts.map
+import * as _forklaunch_core_mappers from '@forklaunch/core/mappers';
+import * as zod from 'zod';
+import * as _sinclair_typebox from '@sinclair/typebox';
+import * as _forklaunch_validator from '@forklaunch/validator';
+
+declare const KafkaWorkerSchemas: <SchemaValidator extends _forklaunch_validator.AnySchemaValidator>(options: Record<string, unknown> & {
+    validator: SchemaValidator;
+}) => _forklaunch_core_mappers.SchemasByValidator<SchemaValidator, (options: Record<string, unknown>) => {
+    brokers: _sinclair_typebox.TArray<_sinclair_typebox.TString>;
+    clientId: _sinclair_typebox.TString;
+    groupId: _sinclair_typebox.TString;
+    retries: _sinclair_typebox.TTransform<_sinclair_typebox.TUnion<[_sinclair_typebox.TNumber, _sinclair_typebox.TString, _sinclair_typebox.TBoolean, _sinclair_typebox.TNull, _sinclair_typebox.TBigInt, _sinclair_typebox.TDate]>, number>;
+    interval: _sinclair_typebox.TTransform<_sinclair_typebox.TUnion<[_sinclair_typebox.TNumber, _sinclair_typebox.TString, _sinclair_typebox.TBoolean, _sinclair_typebox.TNull, _sinclair_typebox.TBigInt, _sinclair_typebox.TDate]>, number>;
+    peekCount: _sinclair_typebox.TTransform<_sinclair_typebox.TUnion<[_sinclair_typebox.TNumber, _sinclair_typebox.TString, _sinclair_typebox.TBoolean, _sinclair_typebox.TNull, _sinclair_typebox.TBigInt, _sinclair_typebox.TDate]>, number>;
+}, (options: Record<string, unknown>) => {
+    brokers: zod.ZodArray<zod.ZodString, "many">;
+    clientId: zod.ZodString;
+    groupId: zod.ZodString;
+    retries: zod.ZodEffects<zod.ZodNumber, number, unknown>;
+    interval: zod.ZodEffects<zod.ZodNumber, number, unknown>;
+    peekCount: zod.ZodEffects<zod.ZodNumber, number, unknown>;
+}>;
+
+export { KafkaWorkerSchemas };