@vercube/queue 1.3.0

package/dist/Strategies/BullMQStrategy.d.mts

@@ -0,0 +1,161 @@
+import { n as QueueTypes, t as QueueStrategy } from "../QueueStrategy-saWSBeU6.mjs";
+import { ConnectionOptions, JobsOptions, QueueOptions, WorkerOptions } from "bullmq";
+//#region src/Strategies/BullMQStrategy.d.ts
+/** Options the BullMQ strategy connects with. */
+export interface BullMQStrategyOptions {
+  /**
+   * Redis connection, either as options or as an existing ioredis client.
+   * @see {@link https://docs.bullmq.io/guide/connections}
+   */
+  connection: ConnectionOptions;
+  /** Key prefix every queue lives under in Redis. */
+  prefix?: string;
+  /** Job options applied to every published job, overridable per job. */
+  defaultJobOptions?: JobsOptions;
+  /** Extra options handed to every `Queue` this strategy creates. */
+  queueOptions?: Omit<QueueOptions, 'connection' | 'prefix' | 'defaultJobOptions'>;
+  /** Extra options handed to every `Worker` this strategy creates. */
+  workerOptions?: Omit<WorkerOptions, 'connection' | 'prefix' | 'concurrency'>;
+}
+/**
+ * BullMQ backed queue implementation.
+ *
+ * Redis keeps the jobs, and BullMQ owns their lifecycle: attempts, backoff,
+ * delays, priorities, progress and the completed and failed sets are all
+ * handled by the broker. Because retries belong to BullMQ, the attempts of a job
+ * are the ones it was published with - `attempts` on `@Job()` is only a fallback
+ * for jobs published without any.
+ *
+ * @example
+ * ```ts
+ * await queueManager.mount({
+ *   strategy: BullMQStrategy,
+ *   initOptions: { connection: { host: '127.0.0.1', port: 6379 } },
+ * });
+ * ```
+ */
+export declare class BullMQStrategy extends QueueStrategy<BullMQStrategyOptions> {
+  /** Transport this strategy talks to. */
+  readonly transport: string;
+  /** Logger instance */
+  private gLogger;
+  /** Options the strategy was initialized with */
+  private fOptions;
+  /** Producers, one per queue name */
+  private fQueues;
+  /** Consumers, one per queue name */
+  private fWorkers;
+  /**
+   * BullMQ supports the full job model natively.
+   *
+   * @returns {QueueTypes.Capabilities} What this strategy supports
+   */
+  get capabilities(): QueueTypes.Capabilities;
+  /**
+   * Stores the connection every queue and worker is created with.
+   * Redis itself is connected lazily by BullMQ, on the first command.
+   *
+   * @param {BullMQStrategyOptions} options - Redis connection and BullMQ defaults
+   * @returns {void}
+   * @throws {QueueError} When no connection is given
+   */
+  initialize(options: BullMQStrategyOptions): void;
+  /**
+   * Adds a job to a BullMQ queue.
+   *
+   * @param {QueueTypes.PublishRequest} request - Job to publish
+   * @returns {Promise<QueueTypes.JobRef>} Reference to the published job
+   * @throws {QueueError} When the job cannot be added
+   */
+  publish(request: QueueTypes.PublishRequest): Promise<QueueTypes.JobRef>;
+  /**
+   * Adds many jobs in a single Redis round trip.
+   *
+   * @param {QueueTypes.PublishRequest[]} requests - Jobs to publish, all on the same queue
+   * @returns {Promise<QueueTypes.JobRef[]>} References to the published jobs
+   * @throws {QueueError} When the jobs cannot be added
+   */
+  publishMany(requests: QueueTypes.PublishRequest[]): Promise<QueueTypes.JobRef[]>;
+  /**
+   * Starts a BullMQ worker on a queue.
+   *
+   * A rejected dispatch is rethrown into BullMQ, which then applies the
+   * attempts and backoff the job was published with.
+   *
+   * @param {QueueTypes.ConsumeRequest} request - Queue to consume, its concurrency and the dispatch callback
+   * @returns {Promise<QueueTypes.ConsumerHandle>} Handle used to stop the worker again
+   * @throws {QueueError} When the worker cannot be started
+   */
+  consume(request: QueueTypes.ConsumeRequest): Promise<QueueTypes.ConsumerHandle>;
+  /**
+   * Reads the job counts BullMQ keeps for a queue.
+   *
+   * @param {string} queue - Queue to read
+   * @returns {Promise<QueueTypes.QueueStats>} Counters of that queue
+   * @throws {QueueError} When the counters cannot be read
+   */
+  stats(queue: string): Promise<QueueTypes.QueueStats>;
+  /**
+   * Shows what a queue is holding, straight from Redis and without consuming
+   * anything. The failed set is the interesting one: BullMQ keeps the data and
+   * the stack trace of every job that ran out of attempts.
+   *
+   * @param {QueueTypes.PeekRequest} request - Queue to look at, how many messages and which states
+   * @returns {Promise<QueueTypes.PeekedMessage[]>} The messages found
+   * @throws {QueueError} When the queue cannot be read
+   */
+  peek(request: QueueTypes.PeekRequest): Promise<QueueTypes.PeekedMessage[]>;
+  /**
+   * Closes every worker and producer this strategy created.
+   *
+   * @returns {Promise<void>} Resolves once everything is closed
+   */
+  close(): Promise<void>;
+  /**
+   * Describes a BullMQ job the way the module reads a peeked message.
+   *
+   * @param {Job} job - The job as Redis holds it
+   * @param {QueueTypes.PeekState} state - Set it was read from
+   * @returns {QueueTypes.PeekedMessage} The message, ready to be inspected
+   */
+  private describe;
+  /**
+   * Returns the producer of a queue, creating it on first use.
+   *
+   * @param {string} name - Queue name
+   * @returns {Queue} The BullMQ queue
+   * @throws {QueueError} When the strategy has not been initialized
+   */
+  private queueFor;
+  /**
+   * Translates the module's job options into BullMQ job options.
+   *
+   * @param {QueueTypes.JobOptions} options - Options of the job being published
+   * @returns {JobsOptions} The BullMQ options
+   */
+  private toJobOptions;
+  /**
+   * Wraps a job so its headers survive the round trip through Redis.
+   *
+   * @param {QueueTypes.PublishRequest} request - Job being published
+   * @returns {BullMQEnvelope} What is stored as the BullMQ job data
+   */
+  private toEnvelope;
+  /**
+   * Reads a job back. Data that is not one of this module's envelopes is treated
+   * as the payload itself, so jobs added by other BullMQ producers still work.
+   *
+   * @param {unknown} data - Data as stored in Redis
+   * @returns {BullMQEnvelope} Payload and headers of the job
+   */
+  private fromEnvelope;
+  /**
+   * Returns the options the strategy was initialized with.
+   *
+   * @param {string} operation - Operation asking for them, reported in the error
+   * @returns {BullMQStrategyOptions} The options
+   * @throws {QueueError} When the strategy has not been initialized
+   */
+  private requireOptions;
+}
+//#endregion

package/dist/Strategies/BullMQStrategy.mjs

@@ -0,0 +1,328 @@
+import { h as prune, t as QueueStrategy, v as QueueError } from "../QueueStrategy-DwLPpfFM.mjs";
+import { n as toQueueError, t as __decorate } from "../decorate-C0p0FnUM.mjs";
+import { InjectOptional } from "@vercube/di";
+import { Logger } from "@vercube/logger";
+import { Queue, UnrecoverableError, Worker } from "bullmq";
+//#region src/Strategies/BullMQStrategy.ts
+/**
+* BullMQ backed queue implementation.
+*
+* Redis keeps the jobs, and BullMQ owns their lifecycle: attempts, backoff,
+* delays, priorities, progress and the completed and failed sets are all
+* handled by the broker. Because retries belong to BullMQ, the attempts of a job
+* are the ones it was published with - `attempts` on `@Job()` is only a fallback
+* for jobs published without any.
+*
+* @example
+* ```ts
+* await queueManager.mount({
+*   strategy: BullMQStrategy,
+*   initOptions: { connection: { host: '127.0.0.1', port: 6379 } },
+* });
+* ```
+*/
+var BullMQStrategy = class extends QueueStrategy {
+	/** Transport this strategy talks to. */
+	transport = "bullmq";
+	/** Logger instance */
+	gLogger;
+	/** Options the strategy was initialized with */
+	fOptions = null;
+	/** Producers, one per queue name */
+	fQueues = /* @__PURE__ */ new Map();
+	/** Consumers, one per queue name */
+	fWorkers = /* @__PURE__ */ new Map();
+	/**
+	* BullMQ supports the full job model natively.
+	*
+	* @returns {QueueTypes.Capabilities} What this strategy supports
+	*/
+	get capabilities() {
+		return {
+			retries: true,
+			delay: true,
+			priority: true,
+			progress: true,
+			stats: true,
+			peek: true
+		};
+	}
+	/**
+	* Stores the connection every queue and worker is created with.
+	* Redis itself is connected lazily by BullMQ, on the first command.
+	*
+	* @param {BullMQStrategyOptions} options - Redis connection and BullMQ defaults
+	* @returns {void}
+	* @throws {QueueError} When no connection is given
+	*/
+	initialize(options) {
+		if (!options?.connection) throw new QueueError("BullMQ needs a Redis connection", "initialize", void 0, void 0, false);
+		this.fOptions = options;
+	}
+	/**
+	* Adds a job to a BullMQ queue.
+	*
+	* @param {QueueTypes.PublishRequest} request - Job to publish
+	* @returns {Promise<QueueTypes.JobRef>} Reference to the published job
+	* @throws {QueueError} When the job cannot be added
+	*/
+	async publish(request) {
+		try {
+			const job = await this.queueFor(request.queue).add(request.job, this.toEnvelope(request), this.toJobOptions(request.options));
+			return {
+				id: String(job.id),
+				queue: request.queue,
+				job: request.job,
+				strategy: this.transport
+			};
+		} catch (error) {
+			throw toQueueError(error, "Failed to add job to BullMQ", "publish", {
+				queue: request.queue,
+				job: request.job
+			});
+		}
+	}
+	/**
+	* Adds many jobs in a single Redis round trip.
+	*
+	* @param {QueueTypes.PublishRequest[]} requests - Jobs to publish, all on the same queue
+	* @returns {Promise<QueueTypes.JobRef[]>} References to the published jobs
+	* @throws {QueueError} When the jobs cannot be added
+	*/
+	async publishMany(requests) {
+		if (requests.length === 0) return [];
+		try {
+			return (await this.queueFor(requests[0].queue).addBulk(requests.map((request) => ({
+				name: request.job,
+				data: this.toEnvelope(request),
+				opts: this.toJobOptions(request.options)
+			})))).map((job, index) => ({
+				id: String(job.id),
+				queue: requests[index].queue,
+				job: requests[index].job,
+				strategy: this.transport
+			}));
+		} catch (error) {
+			throw toQueueError(error, "Failed to add jobs to BullMQ", "publishMany", { queue: requests[0].queue });
+		}
+	}
+	/**
+	* Starts a BullMQ worker on a queue.
+	*
+	* A rejected dispatch is rethrown into BullMQ, which then applies the
+	* attempts and backoff the job was published with.
+	*
+	* @param {QueueTypes.ConsumeRequest} request - Queue to consume, its concurrency and the dispatch callback
+	* @returns {Promise<QueueTypes.ConsumerHandle>} Handle used to stop the worker again
+	* @throws {QueueError} When the worker cannot be started
+	*/
+	async consume(request) {
+		const options = this.requireOptions("consume");
+		const existing = this.fWorkers.get(request.queue);
+		if (existing) await existing.close();
+		const worker = new Worker(request.queue, async (job) => {
+			const { payload, headers } = this.fromEnvelope(job.data);
+			try {
+				await request.dispatch({
+					id: String(job.id),
+					job: job.name,
+					payload,
+					headers,
+					attempt: job.attemptsStarted || job.attemptsMade + 1,
+					attempts: job.opts?.attempts ?? 1,
+					raw: job,
+					updateProgress: (progress) => job.updateProgress(progress)
+				});
+			} catch (error) {
+				if (error instanceof QueueError && !error.retryable) {
+					const cause = error.cause instanceof Error ? `: ${error.cause.message}` : "";
+					throw new UnrecoverableError(`${error.message}${cause}`);
+				}
+				throw error;
+			}
+		}, {
+			...options.workerOptions,
+			connection: options.connection,
+			prefix: options.prefix,
+			concurrency: request.concurrency
+		});
+		worker.on("error", (error) => {
+			this.gLogger?.error(`Vercube/BullMQStrategy::Worker of "${request.queue}" failed`, error);
+		});
+		this.fWorkers.set(request.queue, worker);
+		return {
+			queue: request.queue,
+			stop: async () => {
+				this.fWorkers.delete(request.queue);
+				await worker.close();
+			}
+		};
+	}
+	/**
+	* Reads the job counts BullMQ keeps for a queue.
+	*
+	* @param {string} queue - Queue to read
+	* @returns {Promise<QueueTypes.QueueStats>} Counters of that queue
+	* @throws {QueueError} When the counters cannot be read
+	*/
+	async stats(queue) {
+		try {
+			const counts = await this.queueFor(queue).getJobCounts("waiting", "active", "completed", "failed", "delayed");
+			return {
+				waiting: counts.waiting,
+				active: counts.active,
+				completed: counts.completed,
+				failed: counts.failed,
+				delayed: counts.delayed
+			};
+		} catch (error) {
+			throw toQueueError(error, "Failed to read BullMQ job counts", "stats", { queue });
+		}
+	}
+	/**
+	* Shows what a queue is holding, straight from Redis and without consuming
+	* anything. The failed set is the interesting one: BullMQ keeps the data and
+	* the stack trace of every job that ran out of attempts.
+	*
+	* @param {QueueTypes.PeekRequest} request - Queue to look at, how many messages and which states
+	* @returns {Promise<QueueTypes.PeekedMessage[]>} The messages found
+	* @throws {QueueError} When the queue cannot be read
+	*/
+	async peek(request) {
+		const queue = this.queueFor(request.queue);
+		const messages = [];
+		try {
+			for (const state of request.states) {
+				if (messages.length >= request.limit) break;
+				const jobs = await queue.getJobs([state], 0, request.limit - messages.length - 1);
+				for (const job of jobs) messages.push(this.describe(job, state));
+			}
+		} catch (error) {
+			throw toQueueError(error, "Failed to read the BullMQ queue", "peek", { queue: request.queue });
+		}
+		return messages;
+	}
+	/**
+	* Closes every worker and producer this strategy created.
+	*
+	* @returns {Promise<void>} Resolves once everything is closed
+	*/
+	async close() {
+		const closing = [...[...this.fWorkers.values()].map((worker) => worker.close()), ...[...this.fQueues.values()].map((queue) => queue.close())];
+		this.fWorkers.clear();
+		this.fQueues.clear();
+		await Promise.all(closing);
+	}
+	/**
+	* Describes a BullMQ job the way the module reads a peeked message.
+	*
+	* @param {Job} job - The job as Redis holds it
+	* @param {QueueTypes.PeekState} state - Set it was read from
+	* @returns {QueueTypes.PeekedMessage} The message, ready to be inspected
+	*/
+	describe(job, state) {
+		const { payload, headers } = this.fromEnvelope(job.data);
+		const delay = job.opts?.delay ?? 0;
+		return {
+			id: String(job.id),
+			job: job.name,
+			state,
+			attempt: job.attemptsStarted || job.attemptsMade || void 0,
+			payload,
+			headers,
+			availableAt: state === "delayed" && job.timestamp ? job.timestamp + delay : void 0,
+			error: job.failedReason ? {
+				message: job.failedReason,
+				stack: job.stacktrace?.join("\n") || void 0
+			} : void 0
+		};
+	}
+	/**
+	* Returns the producer of a queue, creating it on first use.
+	*
+	* @param {string} name - Queue name
+	* @returns {Queue} The BullMQ queue
+	* @throws {QueueError} When the strategy has not been initialized
+	*/
+	queueFor(name) {
+		const options = this.requireOptions("publish");
+		let queue = this.fQueues.get(name);
+		if (!queue) {
+			queue = new Queue(name, {
+				...options.queueOptions,
+				connection: options.connection,
+				prefix: options.prefix,
+				defaultJobOptions: options.defaultJobOptions
+			});
+			this.fQueues.set(name, queue);
+		}
+		return queue;
+	}
+	/**
+	* Translates the module's job options into BullMQ job options.
+	*
+	* @param {QueueTypes.JobOptions} options - Options of the job being published
+	* @returns {JobsOptions} The BullMQ options
+	*/
+	toJobOptions(options) {
+		const backoff = options.backoff;
+		return prune({
+			attempts: options.attempts,
+			backoff: typeof backoff === "number" ? {
+				type: "fixed",
+				delay: backoff
+			} : backoff,
+			delay: options.delay,
+			priority: options.priority,
+			jobId: options.jobId,
+			removeOnComplete: options.removeOnComplete,
+			removeOnFail: options.removeOnFail
+		});
+	}
+	/**
+	* Wraps a job so its headers survive the round trip through Redis.
+	*
+	* @param {QueueTypes.PublishRequest} request - Job being published
+	* @returns {BullMQEnvelope} What is stored as the BullMQ job data
+	*/
+	toEnvelope(request) {
+		return {
+			payload: request.payload,
+			headers: request.headers
+		};
+	}
+	/**
+	* Reads a job back. Data that is not one of this module's envelopes is treated
+	* as the payload itself, so jobs added by other BullMQ producers still work.
+	*
+	* @param {unknown} data - Data as stored in Redis
+	* @returns {BullMQEnvelope} Payload and headers of the job
+	*/
+	fromEnvelope(data) {
+		if (data !== null && typeof data === "object" && "payload" in data && "headers" in data) {
+			const envelope = data;
+			return {
+				payload: envelope.payload,
+				headers: envelope.headers ?? {}
+			};
+		}
+		return {
+			payload: data,
+			headers: {}
+		};
+	}
+	/**
+	* Returns the options the strategy was initialized with.
+	*
+	* @param {string} operation - Operation asking for them, reported in the error
+	* @returns {BullMQStrategyOptions} The options
+	* @throws {QueueError} When the strategy has not been initialized
+	*/
+	requireOptions(operation) {
+		if (!this.fOptions) throw new QueueError("BullMQ strategy is not initialized", operation, void 0, void 0, false);
+		return this.fOptions;
+	}
+};
+__decorate([InjectOptional(Logger)], BullMQStrategy.prototype, "gLogger", void 0);
+//#endregion
+export { BullMQStrategy };

package/dist/Strategies/KafkaStrategy.d.mts

@@ -0,0 +1,152 @@
+import { n as QueueTypes, t as QueueStrategy } from "../QueueStrategy-saWSBeU6.mjs";
+import { ConsumerConfig, KafkaConfig, ProducerConfig, ProducerRecord } from "kafkajs";
+//#region src/Strategies/KafkaStrategy.d.ts
+/** Options the Kafka strategy connects with. */
+export interface KafkaStrategyOptions {
+  /** Client configuration: brokers, client id, ssl, sasl and the rest. */
+  client: KafkaConfig;
+  /** Consumer group every consumer of this strategy joins. Required to consume. */
+  groupId?: string;
+  /** Extra producer configuration. */
+  producer?: ProducerConfig;
+  /** Extra consumer configuration, the group id excluded. */
+  consumer?: Omit<ConsumerConfig, 'groupId'>;
+  /** Send options applied to every produced record. */
+  send?: Omit<ProducerRecord, 'topic' | 'messages'>;
+  /**
+   * Start from the earliest offset when the group has none committed yet.
+   * @default false
+   */
+  fromBeginning?: boolean;
+  /**
+   * What to do with a job the manager gave up on. `skip` logs it and commits the
+   * offset, so the partition keeps moving, while `crash` lets the error reach
+   * kafkajs, which stops the consumer.
+   * @default 'skip'
+   */
+  onFailure?: 'skip' | 'crash';
+}
+/**
+ * Kafka backed queue implementation.
+ *
+ * A queue is a topic, the job name travels in the `x-job` header and the payload
+ * is JSON. Kafka is a log rather than a job broker: it has no attempts, delays,
+ * priorities or per-message acknowledgements, so those are handled by the manager
+ * and a job it gives up on only moves the offset forward.
+ *
+ * Ordering is per partition, so use the `key` job option to keep related jobs on
+ * the same partition.
+ *
+ * @example
+ * ```ts
+ * await queueManager.mount({
+ *   strategy: KafkaStrategy,
+ *   initOptions: {
+ *     client: { clientId: 'orders', brokers: ['localhost:9092'] },
+ *     groupId: 'orders-workers',
+ *   },
+ * });
+ * ```
+ */
+export declare class KafkaStrategy extends QueueStrategy<KafkaStrategyOptions> {
+  /** Transport this strategy talks to. */
+  readonly transport: string;
+  /** Logger instance */
+  private gLogger;
+  /** Options the strategy was initialized with */
+  private fOptions;
+  /** The Kafka client */
+  private fKafka;
+  /** The connected producer */
+  private fProducer;
+  /** Admin client, opened only when counters are read */
+  private fAdmin;
+  /** Running consumers, indexed by topic */
+  private fConsumers;
+  /**
+   * Kafka is a log: only the offsets it keeps can be reported, everything else
+   * about the job model is left to the manager.
+   *
+   * @returns {QueueTypes.Capabilities} What this strategy supports
+   */
+  get capabilities(): QueueTypes.Capabilities;
+  /**
+   * Creates the client and connects the producer, so a broken configuration is
+   * reported at boot instead of on the first job.
+   *
+   * @param {KafkaStrategyOptions} options - Client configuration and consumer group
+   * @returns {Promise<void>} Resolves once the producer is connected
+   * @throws {QueueError} When no brokers are given, or the producer cannot connect
+   */
+  initialize(options: KafkaStrategyOptions): Promise<void>;
+  /**
+   * Produces a single record.
+   *
+   * @param {QueueTypes.PublishRequest} request - Job to publish
+   * @returns {Promise<QueueTypes.JobRef>} Reference to the produced record
+   * @throws {QueueError} When the record cannot be produced
+   */
+  publish(request: QueueTypes.PublishRequest): Promise<QueueTypes.JobRef>;
+  /**
+   * Produces many records of the same topic in a single request.
+   *
+   * @param {QueueTypes.PublishRequest[]} requests - Jobs to publish, all on the same topic
+   * @returns {Promise<QueueTypes.JobRef[]>} References to the produced records
+   * @throws {QueueError} When the records cannot be produced
+   */
+  publishMany(requests: QueueTypes.PublishRequest[]): Promise<QueueTypes.JobRef[]>;
+  /**
+   * Subscribes a consumer of the configured group to a topic.
+   *
+   * @param {QueueTypes.ConsumeRequest} request - Topic to consume, its concurrency and the dispatch callback
+   * @returns {Promise<QueueTypes.ConsumerHandle>} Handle used to stop the consumer again
+   * @throws {QueueError} When no consumer group is configured, or the consumer cannot start
+   */
+  consume(request: QueueTypes.ConsumeRequest): Promise<QueueTypes.ConsumerHandle>;
+  /**
+   * Reports how far the consumer group is behind the end of the topic.
+   *
+   * @param {string} queue - Topic to read
+   * @returns {Promise<QueueTypes.QueueStats>} How many records are still to be read
+   * @throws {QueueError} When the offsets cannot be read
+   */
+  stats(queue: string): Promise<QueueTypes.QueueStats>;
+  /**
+   * Disconnects every consumer, the producer and the admin client.
+   *
+   * @returns {Promise<void>} Resolves once everything is disconnected
+   */
+  close(): Promise<void>;
+  /**
+   * Returns the admin client, connecting it on first use.
+   *
+   * @returns {Promise<Admin>} The connected admin client
+   * @throws {QueueError} When the strategy has not been initialized
+   */
+  private adminClient;
+  /**
+   * Returns the options the strategy was initialized with.
+   *
+   * @param {string} operation - Operation asking for them, reported in the error
+   * @returns {KafkaStrategyOptions} The options
+   * @throws {QueueError} When the strategy has not been initialized
+   */
+  private requireOptions;
+  /**
+   * Returns the Kafka client.
+   *
+   * @param {string} operation - Operation asking for it, reported in the error
+   * @returns {Kafka} The client
+   * @throws {QueueError} When the strategy has not been initialized
+   */
+  private requireClient;
+  /**
+   * Returns the connected producer.
+   *
+   * @param {string} operation - Operation asking for it, reported in the error
+   * @returns {Producer} The producer
+   * @throws {QueueError} When the strategy has not been initialized
+   */
+  private requireProducer;
+}
+//#endregion