codeweaver 3.1.2 → 4.0.0

package/src/core/message-broker/bullmq/queue.ts

@@ -0,0 +1,58 @@
+import { Queue, Job } from "bullmq";
+import { ProducerConfig } from "./basic-types";
+
+export class BullQueueProducer<T = any> {
+  private queue?: Queue;
+
+  /**
+   * Creates a new BullMQ producer.
+   * @param config - Configuration options for the producer.
+   */
+  public constructor(private config: ProducerConfig) {}
+
+  /**
+   * Returns the BullMQ queue instance associated with this producer.
+   * If the queue has not been created yet, it will be created with the given connection options.
+   * @returns The BullMQ queue instance associated with this producer.
+   */
+  private getQueue(): Queue {
+    if (!this.queue) {
+      this.queue = new Queue(this.config.queueName, {
+        connection: this.config.connection ?? undefined,
+      });
+    }
+    return this.queue!;
+  }
+
+  /**
+   * Adds a job to the queue.
+   * If a serializer is provided in the producer config, it will be used to serialize the data.
+   * Otherwise, the data will be passed as-is.
+   * @param data - The data to serialize and add to the queue.
+   * @param opts - Optional job options to add to the job.
+   * @returns A promise resolved with the added job.
+   */
+  public async addJob(data: T, opts?: any): Promise<Job> {
+    // Serialize data if serializer provided, but Bull's data field is passed as is
+    const payload = this.config.serializer?.serialize
+      ? this.config.serializer.serialize(data)
+      : data;
+
+    const queue = this.getQueue();
+    // You can pass the raw data; a serializer is mainly for transport
+    return await queue.add("job", payload, {
+      ...(this.config.defaultJobOptions ?? {}),
+      ...opts,
+    });
+  }
+
+  /**
+   * Closes the BullMQ queue associated with this producer.
+   * If the queue has not been created yet, this method does nothing.
+   * @returns A promise resolved when the queue has been closed.
+   */
+  public async close(): Promise<void> {
+    await this.queue?.close?.();
+    this.queue = undefined;
+  }
+}

package/src/core/message-broker/bullmq/worker.ts

@@ -0,0 +1,68 @@
+import { Worker, Job } from "bullmq";
+import { ConsumerConfig } from "./basic-types";
+
+export class BullWorker<T = any> {
+  private worker?: Worker;
+  private config: ConsumerConfig;
+
+  /**
+   * Constructor for a BullWorker.
+   * @param config - Configuration options for the worker.
+   */
+  constructor(config: ConsumerConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Returns the BullMQ worker associated with this worker.
+   * If the worker has not been created yet, it will be created with the given connection options.
+   * @returns The BullMQ worker associated with this worker.
+   */
+  private getWorker(): Worker {
+    if (!this.worker) {
+      this.worker = new Worker(
+        this.config.queueName,
+        async (job: Job) => {
+          const data = this.config.serializer?.deserialize
+            ? (this.config.serializer.deserialize(job.data) as T)
+            : (job.data as T);
+          await this.config.processor({
+            id: job.id,
+            data,
+            timestamp: Date.now(),
+            attemptsMade: job.attemptsMade,
+          });
+        },
+        {
+          connection: this.config.connection ?? undefined,
+          concurrency: this.config.concurrency ?? 1,
+        }
+      );
+    }
+    return this.worker;
+  }
+
+  /**
+   * Starts the BullMQ worker.
+   * The worker will process all messages published to the associated queue.
+   * If the worker has not been created yet, it will be created with the given connection options.
+   * @returns A promise resolved when the worker has been started.
+   */
+  public async start(): Promise<void> {
+    this.getWorker();
+
+    this.worker?.on("error", (err) => {
+      throw err;
+    });
+  }
+
+  /**
+   * Stops the BullMQ worker.
+   * If the worker has not been created yet, this method does nothing.
+   * @returns A promise resolved when the worker has been stopped.
+   */
+  public async stop(): Promise<void> {
+    await this.worker?.close?.();
+    this.worker = undefined;
+  }
+}

package/src/core/message-broker/kafka/basic-types.ts

@@ -0,0 +1,45 @@
+import { EachMessagePayload } from "kafkajs";
+
+export type KafkaKey = string | Buffer;
+export type KafkaHeaders = Record<string, string | Buffer>;
+
+/**
+ * Kafka producer config
+ * @see https://kafka.js.org/docs/producer
+ */
+export interface ProducerConfig {
+  clientId?: string;
+  brokers: string[]; // ["host:port", ...]
+  retry?: {
+    retries?: number;
+    initialRetryTime?: number;
+    backoffMax?: number;
+  };
+  // enableIdempotence is available on producer in kafkajs
+  idempotent?: boolean;
+  // acks: -1, 1, 0
+  allowAutoTopicCreation?: boolean;
+}
+
+/**
+ * Kafka message
+ */
+export interface Message<T = any> {
+  key?: KafkaKey;
+  value: T;
+  headers?: KafkaHeaders;
+  partition?: number;
+  timestamp?: string;
+}
+
+/**
+ * Kafka consumer config
+ * @see https://kafka.js.org/docs/consumer
+ */
+export interface ConsumerConfig {
+  clientId?: string;
+  brokers: string[];
+  groupId: string;
+  fromBeginning?: boolean;
+  eachMessage?: (payload: EachMessagePayload) => Promise<void> | void;
+}

package/src/core/message-broker/kafka/consumer.ts

@@ -0,0 +1,95 @@
+import { Kafka, Consumer, EachMessagePayload } from "kafkajs";
+import { ConsumerConfig } from "./basic-types";
+import { BrokerSerializer, getDefaultJsonSchema } from "../utilities";
+
+/**
+ * Kafka consumer
+ * @see https://kafka.js.org/docs/consumer
+ */
+export class KafkaConsumer<T> {
+  private consumer?: Consumer;
+  private client: Kafka;
+
+  /**
+   * Creates a new Kafka consumer.
+   * @param config - Configuration options for the consumer.
+   * @param schema - Optional serializer for deserializing messages.
+   * @param messageHandler - Optional message handler to invoke when a message is received.
+   */
+  public constructor(
+    public config: ConsumerConfig,
+    public schema?: BrokerSerializer<T>,
+    public messageHandler?: (
+      value: T,
+      payload: EachMessagePayload
+    ) => Promise<void> | void
+  ) {
+    this.client = new Kafka({
+      clientId: config.clientId ?? "kafka-consumer",
+      brokers: config.brokers,
+    });
+    this.schema = schema ?? getDefaultJsonSchema<T>();
+  }
+
+  /**
+   * Connects to the Kafka cluster.
+   * If the consumer is already connected, this method returns immediately.
+   * Otherwise, it creates a new consumer and connects to the Kafka cluster.
+   * @returns A promise resolved when the consumer is connected.
+   */
+  public async connect(): Promise<void> {
+    if (this.consumer) return;
+    this.consumer = this.client.consumer({ groupId: this.config.groupId });
+    await this.consumer.connect();
+  }
+
+  /**
+   * Subscribes to the given topics.
+   * If the consumer is not already connected, it will first connect to the Kafka cluster.
+   * By default, the consumer will subscribe to the topics from the beginning.
+   * This can be overridden by passing "fromBeginning" in the config.
+   * @param topics - The topics to subscribe to.
+   * @returns A promise resolved when the consumer is subscribed to the topics.
+   */
+  public async subscribeTopics(topics: string[]): Promise<void> {
+    if (!this.consumer) await this.connect();
+    for (const t of topics) {
+      // subscribe from beginning by default; can be overridden via eachTopic
+      await this.consumer.subscribe({
+        topic: t,
+        fromBeginning: this.config.fromBeginning ?? false,
+      });
+    }
+  }
+
+  /**
+   * Starts the consumer and begins consuming messages from the subscribed topics.
+   * If the consumer is not already connected, it will first connect to the Kafka cluster.
+   * @throws {Error} If the consumer is not connected.
+   */
+  public async run(): Promise<void> {
+    if (!this.consumer)
+      throw new Error("Consumer not connected. Call connect() first.");
+
+    await this.consumer.run({
+      eachMessage: async (payload: EachMessagePayload) => {
+        await this.messageHandler?.(
+          this.schema.deserialize(payload.message.value),
+          payload
+        );
+      },
+    });
+  }
+
+  /**
+   * Disconnects from the Kafka cluster and unsubscribes from all topics.
+   * If the consumer is not currently connected, this method does nothing.
+   * @returns A promise resolved when the consumer is disconnected.
+   */
+  public async disconnect(): Promise<void> {
+    if (this.consumer) {
+      await this.consumer.disconnect();
+      this.consumer = undefined;
+    }
+  }
+}

package/src/core/message-broker/kafka/index.ts

@@ -0,0 +1,3 @@
+export * from "./basic-types";
+export * from "./producer";
+export * from "./consumer";

package/src/core/message-broker/kafka/producer.ts

@@ -0,0 +1,113 @@
+import { Kafka, Producer, ProducerRecord, KafkaConfig } from "kafkajs";
+import { ProducerConfig, Message } from "./basic-types";
+import { backoff, getDefaultJsonSchema, BrokerSerializer } from "../utilities";
+
+/**
+ * Kafka producer
+ * @see https://kafka.js.org/docs/producer
+ */
+export class KafkaProducer<T = any> {
+  private producer?: Producer;
+  private client: Kafka;
+
+  /**
+   * Creates a new Kafka producer.
+   * @param config - Configuration options for the producer.
+   * @param schema - Optional serializer for deserializing messages.
+   * @see https://kafka.js.org/docs/producer
+   */
+  public constructor(
+    public config: ProducerConfig,
+    public schema?: BrokerSerializer<T>
+  ) {
+    this.schema = schema ?? getDefaultJsonSchema<T>();
+    const kafkaConfig: KafkaConfig = {
+      clientId: config.clientId ?? "kafka-client",
+      brokers: config.brokers,
+      retry: {
+        retries: config.retry?.retries ?? 1,
+        initialRetryTime: config.retry?.initialRetryTime ?? 300,
+        maxRetryTime: config.retry?.backoffMax ?? 1000,
+      },
+    };
+
+    this.client = new Kafka(kafkaConfig);
+  }
+
+  /**
+   * Connects to the Kafka cluster.
+   * If the producer is already connected, this method returns immediately.
+   * Otherwise, it creates a new producer and connects to the Kafka cluster.
+   * @returns A promise resolved when the producer is connected.
+   */
+  public async connect(): Promise<void> {
+    if (this.producer) return;
+    this.producer = this.client.producer({
+      idempotent: this.config.idempotent ?? true,
+      allowAutoTopicCreation: this.config.allowAutoTopicCreation ?? true,
+    });
+    await this.producer.connect();
+  }
+
+  /**
+   * Disconnects from the Kafka cluster.
+   * If the producer is not currently connected, this method does nothing.
+   * @returns A promise resolved when the producer is disconnected.
+   */
+  public async disconnect(): Promise<void> {
+    if (this.producer) {
+      await this.producer.disconnect();
+      this.producer = undefined;
+    }
+  }
+
+  /**
+   * Sends a message to the Kafka cluster.
+   * If the producer is not currently connected, this method will first connect to the Kafka cluster.
+   * The message will be serialized using the provided schema.
+   * If the message key is provided, it will be used to determine the partition to send the message to.
+   * If the message headers are provided, they will be sent with the message.
+   * If the message partition is provided, the message will be sent to that partition.
+   * If the message timestamp is provided, it will be used as the timestamp for the message.
+   * In case of transient failures, the method will retry up to the configured number of retries.
+   * @param topic - The topic to send the message to.
+   * @param message - The message to send.
+   * @returns A promise resolved when the message has been sent to the topic.
+   */
+  public async send(topic: string, message: Message<T>): Promise<void> {
+    if (!this.producer) await this.connect();
+
+    const valueBuf = this.schema.serialize(message.value);
+    const payload: ProducerRecord = {
+      topic,
+      messages: [
+        {
+          key: message.key
+            ? typeof message.key === "string"
+              ? message.key
+              : message.key.toString()
+            : undefined,
+          value: valueBuf,
+          headers: message.headers,
+          partition: message.partition,
+          timestamp: message.timestamp,
+        },
+      ],
+    };
+
+    // Simple retry wrapper with backoff in case of transient failures
+    let attempt = 0;
+    while (true) {
+      try {
+        await this.producer!.send(payload);
+        return;
+      } catch (e) {
+        attempt++;
+        const max = this.config.retry?.retries ?? 1;
+        if (attempt > max) throw e;
+        const wait = backoff(attempt, 100, 3000);
+        await new Promise((r) => setTimeout(r, wait));
+      }
+    }
+  }
+}

package/src/core/message-broker/rabitmq/basic-types.ts

@@ -0,0 +1,44 @@
+import { BrokerSerializer } from "../utilities";
+
+/**
+ * Supported RabbitMQ exchange types.
+ */
+export type ExchangeType = "direct" | "fanout" | "topic" | "headers";
+
+/**
+ * Configuration options for a RabbitMQ producer.
+ */
+export interface ProducerConfig<T = any> {
+  url: string;
+  exchange?: string;
+  exchangeType?: ExchangeType;
+  routingKey?: string;
+  confirm?: boolean;
+  retry?: {
+    retries?: number;
+    baseMs?: number;
+    maxMs?: number;
+  };
+  serializer?: BrokerSerializer<T>;
+}
+
+/**
+ * Configuration options for a RabbitMQ consumer.
+ */
+export interface ConsumerConfig<T = any> {
+  url: string;
+  queue: string;
+  exchange?: string;
+  exchangeType?: ExchangeType;
+  routingKey?: string;
+  prefetch?: number;
+  durable?: boolean;
+  onMessage: (msg: {
+    content: T;
+    fields: any;
+    properties: any;
+  }) => Promise<void> | void;
+  ackMode?: "auto" | "manual";
+  retry?: { retries?: number; baseMs?: number; maxMs?: number };
+  serializer?: BrokerSerializer<T>;
+}

package/src/core/message-broker/rabitmq/channel.ts

@@ -0,0 +1,95 @@
+import amqplib, { Connection, Channel } from "amqplib";
+
+/**
+ * Manages a RabbitMQ channel and connection.
+ */
+export class ChannelManager {
+  private connection?: Connection;
+  private channel?: Channel;
+
+  /**
+   * Creates a new ChannelManager.
+   * @param opts - Connection options.
+   * @param opts.url - The URL of the RabbitMQ server.
+   */
+  public constructor(private opts: { url: string }) {}
+
+  /**
+   * Establishes a connection to the RabbitMQ server and creates a new channel.
+   * If a connection has already been established, this method returns immediately.
+   * @returns A promise resolved when the connection has been established.
+   */
+  public async connect(): Promise<void> {
+    if (this.connection) return;
+    this.connection = await amqplib.connect(this.opts.url);
+    this.channel = await this.connection.createChannel();
+  }
+
+  /**
+   * Returns the underlying Channel object.
+   * Throws if connect() has not been called first.
+   * @returns The underlying Channel object.
+   */
+  public get ch(): Channel {
+    if (!this.channel)
+      throw new Error("Channel not initialized. Call connect() first.");
+    return this.channel;
+  }
+
+  /**
+   * Closes the underlying Channel and Connection objects.
+   * If the Channel or Connection objects do not exist, this method does nothing.
+   * @returns A promise resolved when the Channel and Connection objects have been closed.
+   */
+  public async close(): Promise<void> {
+    await this.channel?.close?.();
+    await this.connection?.close?.();
+    this.channel = undefined;
+    this.connection = undefined;
+  }
+
+  /**
+   * Asserts the existence of a queue on the RabbitMQ server.
+   * @param queue - The name of the queue to assert.
+   * @param durable - Optional flag indicating whether the queue should be durable.
+   * @returns A promise resolved when the queue has been asserted.
+   * @throws If the queue does not exist or if the durable flag does not match the queue's durable status.
+   */
+  public async assertQueue(
+    queue: string,
+    durable: boolean = true
+  ): Promise<void> {
+    await this.ch.assertQueue(queue, { durable });
+  }
+
+  /**
+   * Asserts the existence of an exchange on the RabbitMQ server.
+   * @param name - The name of the exchange to assert.
+   * @param type - The type of the exchange to assert.
+   * @param durable - Optional flag indicating whether the exchange should be durable.
+   * @returns A promise resolved when the exchange has been asserted.
+   * @throws If the exchange does not exist or if the durable flag does not match the exchange's durable status.
+   */
+  public async assertExchange(
+    name: string,
+    type: string,
+    durable: boolean = true
+  ): Promise<void> {
+    await this.ch.assertExchange(name, type, { durable });
+  }
+
+  /**
+   * Binds a queue to an exchange with an optional routing key.
+   * @param queue - The name of the queue to bind.
+   * @param exchange - The name of the exchange to bind to.
+   * @param routingKey - Optional routing key to use when binding the queue to the exchange.
+   * @returns A promise resolved when the queue has been bound to the exchange.
+   */
+  public async bindQueue(
+    queue: string,
+    exchange: string,
+    routingKey: string = ""
+  ): Promise<void> {
+    await this.ch.bindQueue(queue, exchange, routingKey);
+  }
+}

package/src/core/message-broker/rabitmq/consumer.ts

@@ -0,0 +1,94 @@
+import { ChannelManager } from "./channel";
+import { ConsumerConfig } from "./basic-types";
+
+/**
+ * RabbitMQ consumer.
+ * @template T - The type of the message body.
+ */
+export class RabbitConsumer<T> {
+  private cm = new ChannelManager({ url: "" });
+  private config: ConsumerConfig<T>;
+  private messageHandler?: (msg: {
+    content: T;
+    fields: any;
+    properties: any;
+  }) => Promise<void> | void;
+
+  /**
+   * Creates a new RabbitMQ consumer.
+   * @param opts - Configuration options for the consumer.
+   */
+  public constructor(opts: ConsumerConfig<T>) {
+    this.config = opts;
+    this.cm = new ChannelManager({ url: opts.url });
+  }
+
+  /**
+   * Connects to the RabbitMQ cluster and sets up the consumer.
+   * Ensures the queue and exchange exist and binds the queue to the exchange.
+   * @returns A promise resolved when the consumer is connected.
+   */
+  public async connect(): Promise<void> {
+    await this.cm.connect();
+    // ensure queue exists
+    await this.cm.assertQueue(this.config.queue, true);
+    if (this.config.exchange) {
+      const type = this.config.exchangeType ?? "direct";
+      // ensure exchange exists
+      await this.cm.assertExchange(this.config.exchange, type, true);
+      // bind queue
+      const routingKey = this.config.routingKey ?? "";
+      await this.cm.bindQueue(
+        this.config.queue,
+        this.config.exchange,
+        routingKey
+      );
+    }
+  }
+
+  /**
+   * Starts the consumer and begins consuming messages from the subscribed queue.
+   * If the consumer is not already connected, it will first connect to the RabbitMQ cluster.
+   * @param prefetch - Optional prefetch count to set on the channel.
+   * If not specified, defaults to the value of `prefetch` in the consumer config.
+   * @returns A promise resolved when the consumer is started.
+   * @throws If the queue is not specified in the consumer config.
+   */
+  public async start(prefetch = this.config.prefetch ?? 1): Promise<void> {
+    if (!this.config.queue) throw new Error("Queue must be specified");
+    await this.connect();
+    await this.cm.ch.prefetch(prefetch);
+    await this.cm.ch.consume(this.config.queue, async (msg) => {
+      if (!msg) return;
+      // Deserialize message body
+      let content: T;
+      if (this.config?.serializer?.deserialize) {
+        content = this.config.serializer.deserialize(msg.content) as T;
+      } else {
+        content = JSON.parse(msg.content.toString()) as T;
+      }
+      const payload = {
+        content,
+        fields: msg.fields,
+        properties: msg.properties,
+      };
+
+      if (this.messageHandler) await this.messageHandler(payload);
+
+      // Acknowledge in either auto or manual mode
+      if (this.config.ackMode !== "manual") {
+        this.cm.ch.ack(msg);
+      }
+    });
+  }
+
+  /**
+   * Stops the consumer and closes the underlying Channel and Connection objects.
+   * The consumer will no longer receive messages from the subscribed queue.
+   * @returns A promise resolved when the consumer has been stopped.
+   */
+  public async stop(): Promise<void> {
+    // Rudimentary stop: Cancel consumption by closing channel/connection
+    await this.cm.close();
+  }
+}

package/src/core/message-broker/rabitmq/index.ts

@@ -0,0 +1,4 @@
+export * from "./basic-types";
+export * from "./channel";
+export * from "./producer";
+export * from "./consumer";