@vercube/queue 1.3.0

package/dist/Strategies/KafkaStrategy.mjs

@@ -0,0 +1,280 @@
+import { f as encodePayload, g as readNumericHeader, m as normalizeHeaders, r as ATTEMPT_HEADER, t as QueueStrategy, u as decodePayload, v as QueueError } from "../QueueStrategy-DwLPpfFM.mjs";
+import { n as toQueueError, t as __decorate } from "../decorate-C0p0FnUM.mjs";
+import { InjectOptional } from "@vercube/di";
+import { Logger } from "@vercube/logger";
+import { Kafka } from "kafkajs";
+//#region src/Strategies/KafkaStrategy.ts
+/**
+* Kafka backed queue implementation.
+*
+* A queue is a topic, the job name travels in the `x-job` header and the payload
+* is JSON. Kafka is a log rather than a job broker: it has no attempts, delays,
+* priorities or per-message acknowledgements, so those are handled by the manager
+* and a job it gives up on only moves the offset forward.
+*
+* Ordering is per partition, so use the `key` job option to keep related jobs on
+* the same partition.
+*
+* @example
+* ```ts
+* await queueManager.mount({
+*   strategy: KafkaStrategy,
+*   initOptions: {
+*     client: { clientId: 'orders', brokers: ['localhost:9092'] },
+*     groupId: 'orders-workers',
+*   },
+* });
+* ```
+*/
+var KafkaStrategy = class extends QueueStrategy {
+	/** Transport this strategy talks to. */
+	transport = "kafka";
+	/** Logger instance */
+	gLogger;
+	/** Options the strategy was initialized with */
+	fOptions = null;
+	/** The Kafka client */
+	fKafka = null;
+	/** The connected producer */
+	fProducer = null;
+	/** Admin client, opened only when counters are read */
+	fAdmin = null;
+	/** Running consumers, indexed by topic */
+	fConsumers = /* @__PURE__ */ new Map();
+	/**
+	* Kafka is a log: only the offsets it keeps can be reported, everything else
+	* about the job model is left to the manager.
+	*
+	* @returns {QueueTypes.Capabilities} What this strategy supports
+	*/
+	get capabilities() {
+		return {
+			retries: false,
+			delay: false,
+			priority: false,
+			progress: false,
+			stats: true,
+			peek: false
+		};
+	}
+	/**
+	* Creates the client and connects the producer, so a broken configuration is
+	* reported at boot instead of on the first job.
+	*
+	* @param {KafkaStrategyOptions} options - Client configuration and consumer group
+	* @returns {Promise<void>} Resolves once the producer is connected
+	* @throws {QueueError} When no brokers are given, or the producer cannot connect
+	*/
+	async initialize(options) {
+		if (!options?.client?.brokers) throw new QueueError("Kafka needs at least one broker", "initialize", void 0, void 0, false);
+		this.fOptions = options;
+		this.fKafka = new Kafka(options.client);
+		try {
+			this.fProducer = this.fKafka.producer(options.producer);
+			await this.fProducer.connect();
+		} catch (error) {
+			this.fProducer = null;
+			throw toQueueError(error, "Failed to connect the Kafka producer", "initialize");
+		}
+	}
+	/**
+	* Produces a single record.
+	*
+	* @param {QueueTypes.PublishRequest} request - Job to publish
+	* @returns {Promise<QueueTypes.JobRef>} Reference to the produced record
+	* @throws {QueueError} When the record cannot be produced
+	*/
+	async publish(request) {
+		const [ref] = await this.publishMany([request]);
+		return ref;
+	}
+	/**
+	* Produces many records of the same topic in a single request.
+	*
+	* @param {QueueTypes.PublishRequest[]} requests - Jobs to publish, all on the same topic
+	* @returns {Promise<QueueTypes.JobRef[]>} References to the produced records
+	* @throws {QueueError} When the records cannot be produced
+	*/
+	async publishMany(requests) {
+		if (requests.length === 0) return [];
+		const producer = this.requireProducer("publish");
+		const topic = requests[0].queue;
+		try {
+			const metadata = await producer.send({
+				...this.fOptions?.send,
+				topic,
+				messages: requests.map((request) => ({
+					key: request.options.key ?? request.options.jobId ?? null,
+					value: encodePayload(request.payload),
+					headers: request.headers
+				}))
+			});
+			const partition = metadata[0]?.partition ?? 0;
+			const baseOffset = Number(metadata[0]?.baseOffset ?? 0);
+			return requests.map((request, index) => ({
+				id: `${topic}-${partition}-${baseOffset + index}`,
+				queue: topic,
+				job: request.job,
+				strategy: this.transport
+			}));
+		} catch (error) {
+			throw toQueueError(error, "Failed to produce Kafka records", "publish", { queue: topic });
+		}
+	}
+	/**
+	* Subscribes a consumer of the configured group to a topic.
+	*
+	* @param {QueueTypes.ConsumeRequest} request - Topic to consume, its concurrency and the dispatch callback
+	* @returns {Promise<QueueTypes.ConsumerHandle>} Handle used to stop the consumer again
+	* @throws {QueueError} When no consumer group is configured, or the consumer cannot start
+	*/
+	async consume(request) {
+		const options = this.requireOptions("consume");
+		const kafka = this.requireClient("consume");
+		if (!options.groupId) throw new QueueError("Kafka needs a groupId to consume a topic", "consume", void 0, { queue: request.queue }, false);
+		const previous = this.fConsumers.get(request.queue);
+		if (previous) {
+			this.fConsumers.delete(request.queue);
+			await previous.disconnect().catch(() => void 0);
+		}
+		const consumer = kafka.consumer({
+			...options.consumer,
+			groupId: options.groupId
+		});
+		try {
+			await consumer.connect();
+			await consumer.subscribe({
+				topic: request.queue,
+				fromBeginning: options.fromBeginning ?? false
+			});
+			await consumer.run({
+				partitionsConsumedConcurrently: Math.max(1, request.concurrency),
+				eachMessage: async ({ partition, message }) => {
+					const headers = normalizeHeaders(message.headers);
+					try {
+						await request.dispatch({
+							id: `${request.queue}-${partition}-${message.offset}`,
+							job: headers["x-job"] ?? "unknown",
+							payload: decodePayload(message.value),
+							headers,
+							attempt: readNumericHeader(headers[ATTEMPT_HEADER], 1),
+							raw: message
+						});
+					} catch (error) {
+						if ((options.onFailure ?? "skip") === "crash") throw error;
+						this.gLogger?.error(`Vercube/KafkaStrategy::Skipping a failed job of "${request.queue}"`, error);
+					}
+				}
+			});
+		} catch (error) {
+			await consumer.disconnect().catch(() => void 0);
+			throw toQueueError(error, "Failed to consume the Kafka topic", "consume", { queue: request.queue });
+		}
+		this.fConsumers.set(request.queue, consumer);
+		return {
+			queue: request.queue,
+			stop: async () => {
+				if (this.fConsumers.get(request.queue) === consumer) this.fConsumers.delete(request.queue);
+				await consumer.disconnect();
+			}
+		};
+	}
+	/**
+	* Reports how far the consumer group is behind the end of the topic.
+	*
+	* @param {string} queue - Topic to read
+	* @returns {Promise<QueueTypes.QueueStats>} How many records are still to be read
+	* @throws {QueueError} When the offsets cannot be read
+	*/
+	async stats(queue) {
+		const options = this.requireOptions("stats");
+		if (!options.groupId) return {};
+		try {
+			const admin = await this.adminClient();
+			const [ends, committed] = await Promise.all([admin.fetchTopicOffsets(queue), admin.fetchOffsets({
+				groupId: options.groupId,
+				topics: [queue]
+			})]);
+			const offsets = new Map(committed[0]?.partitions.map((entry) => [entry.partition, Number(entry.offset)]) ?? []);
+			let waiting = 0;
+			for (const end of ends) {
+				const at = Math.max(0, offsets.get(end.partition) ?? 0);
+				waiting += Math.max(0, Number(end.offset) - at);
+			}
+			return { waiting };
+		} catch (error) {
+			throw toQueueError(error, "Failed to read Kafka offsets", "stats", { queue });
+		}
+	}
+	/**
+	* Disconnects every consumer, the producer and the admin client.
+	*
+	* @returns {Promise<void>} Resolves once everything is disconnected
+	*/
+	async close() {
+		const consumers = [...this.fConsumers.values()];
+		const producer = this.fProducer;
+		const admin = this.fAdmin;
+		this.fConsumers.clear();
+		this.fProducer = null;
+		this.fAdmin = null;
+		try {
+			await Promise.all([
+				...consumers.map((consumer) => consumer.disconnect()),
+				producer?.disconnect(),
+				admin?.disconnect()
+			]);
+		} catch (error) {
+			this.gLogger?.warn("Vercube/KafkaStrategy::Failed to disconnect", error);
+		}
+	}
+	/**
+	* Returns the admin client, connecting it on first use.
+	*
+	* @returns {Promise<Admin>} The connected admin client
+	* @throws {QueueError} When the strategy has not been initialized
+	*/
+	async adminClient() {
+		if (!this.fAdmin) {
+			this.fAdmin = this.requireClient("stats").admin();
+			await this.fAdmin.connect();
+		}
+		return this.fAdmin;
+	}
+	/**
+	* Returns the options the strategy was initialized with.
+	*
+	* @param {string} operation - Operation asking for them, reported in the error
+	* @returns {KafkaStrategyOptions} The options
+	* @throws {QueueError} When the strategy has not been initialized
+	*/
+	requireOptions(operation) {
+		if (!this.fOptions) throw new QueueError("Kafka strategy is not initialized", operation, void 0, void 0, false);
+		return this.fOptions;
+	}
+	/**
+	* Returns the Kafka client.
+	*
+	* @param {string} operation - Operation asking for it, reported in the error
+	* @returns {Kafka} The client
+	* @throws {QueueError} When the strategy has not been initialized
+	*/
+	requireClient(operation) {
+		if (!this.fKafka) throw new QueueError("Kafka strategy is not initialized", operation, void 0, void 0, false);
+		return this.fKafka;
+	}
+	/**
+	* Returns the connected producer.
+	*
+	* @param {string} operation - Operation asking for it, reported in the error
+	* @returns {Producer} The producer
+	* @throws {QueueError} When the strategy has not been initialized
+	*/
+	requireProducer(operation) {
+		if (!this.fProducer) throw new QueueError("Kafka strategy is not initialized", operation, void 0, void 0, false);
+		return this.fProducer;
+	}
+};
+__decorate([InjectOptional(Logger)], KafkaStrategy.prototype, "gLogger", void 0);
+//#endregion
+export { KafkaStrategy };

package/dist/Strategies/MemoryStrategy.d.mts

@@ -0,0 +1,139 @@
+import { n as QueueTypes, t as QueueStrategy } from "../QueueStrategy-saWSBeU6.mjs";
+//#region src/Strategies/MemoryStrategy.d.ts
+/**
+ * In-process queue implementation.
+ *
+ * Jobs never leave the running process, which makes this the strategy to use in
+ * tests, in examples and for background work that may be lost on restart. It is
+ * also the reference implementation: delays, priorities and progress all behave
+ * exactly as the module documents them.
+ *
+ * @example
+ * ```ts
+ * await queueManager.mount({ strategy: MemoryStrategy });
+ * ```
+ */
+export declare class MemoryStrategy extends QueueStrategy {
+  /** Transport this strategy talks to. */
+  readonly transport: string;
+  /** Queues created so far, indexed by name */
+  private fQueues;
+  /** Monotonic counter keeping jobs of equal priority in publish order */
+  private fSequence;
+  /** Callbacks waiting for a condition on the queues to hold */
+  private fWaiters;
+  /**
+   * Everything the module offers is supported, jobs simply live in memory.
+   *
+   * @returns {QueueTypes.Capabilities} What this strategy supports
+   */
+  get capabilities(): QueueTypes.Capabilities;
+  /**
+   * Nothing to connect to.
+   *
+   * @returns {void}
+   */
+  initialize(): void;
+  /**
+   * Adds a job to an in-memory queue, honouring its delay and priority.
+   *
+   * @param {QueueTypes.PublishRequest} request - Job to publish
+   * @returns {Promise<QueueTypes.JobRef>} Reference to the published job
+   */
+  publish(request: QueueTypes.PublishRequest): Promise<QueueTypes.JobRef>;
+  /**
+   * Starts processing a queue. Only one consumer per queue is kept, a second
+   * call replaces the first.
+   *
+   * @param {QueueTypes.ConsumeRequest} request - Queue to consume, its concurrency and the dispatch callback
+   * @returns {Promise<QueueTypes.ConsumerHandle>} Handle used to stop the consumer again
+   */
+  consume(request: QueueTypes.ConsumeRequest): Promise<QueueTypes.ConsumerHandle>;
+  /**
+   * Reads the counters of a queue.
+   *
+   * @param {string} name - Queue to read
+   * @returns {Promise<QueueTypes.QueueStats>} Counters of that queue
+   */
+  stats(name: string): Promise<QueueTypes.QueueStats>;
+  /**
+   * Shows what a queue is holding: the jobs waiting to run and the ones still
+   * waiting for their delay.
+   *
+   * @param {QueueTypes.PeekRequest} request - Queue to look at, how many messages and which states
+   * @returns {Promise<QueueTypes.PeekedMessage[]>} The messages found, in the order they would run
+   */
+  peek(request: QueueTypes.PeekRequest): Promise<QueueTypes.PeekedMessage[]>;
+  /**
+   * Drops every queue, pending job and delay timer.
+   *
+   * @returns {Promise<void>} Resolves once everything is dropped
+   */
+  close(): Promise<void>;
+  /**
+   * Waits until no job is waiting, delayed or running.
+   * Handy in tests, where publishing and processing happen in the same process.
+   *
+   * @returns {Promise<void>} Resolves once every queue ran dry
+   */
+  idle(): Promise<void>;
+  /**
+   * Hands as many waiting jobs to the consumer as its concurrency allows.
+   *
+   * @param {string} name - Queue to pump
+   * @returns {void}
+   */
+  private pump;
+  /**
+   * Runs a single job and keeps the queue moving afterwards.
+   *
+   * A rejected dispatch means the job failed for good: the manager has already
+   * applied the retry policy, so nothing is put back on the queue.
+   *
+   * @param {string} name - Queue the job belongs to
+   * @param {MemoryQueue} queue - State of that queue
+   * @param {MemoryJob} job - Job to run
+   * @returns {Promise<void>} Resolves once the job settled
+   */
+  private run;
+  /**
+   * Describes a job the way the module reads a peeked message.
+   *
+   * @param {MemoryJob} job - Job sitting on the queue
+   * @param {QueueTypes.PeekState} state - Where it is sitting
+   * @returns {QueueTypes.PeekedMessage} The message, ready to be inspected
+   */
+  private describe;
+  /**
+   * Picks the job to run next: lowest priority value first, publish order otherwise.
+   *
+   * @param {MemoryQueue} queue - Queue to take from
+   * @returns {MemoryJob | undefined} The next job, or undefined when the queue is empty
+   */
+  private takeNext;
+  /**
+   * Returns the state of a queue, creating it on first use.
+   *
+   * @param {string} name - Queue name
+   * @returns {MemoryQueue} State of that queue
+   */
+  private queueFor;
+  /**
+   * @returns {boolean} True when no job is waiting, delayed or running
+   */
+  private isIdle;
+  /**
+   * Waits until a condition on the queues holds.
+   *
+   * @param {() => boolean} ready - Condition to wait for, checked after every job settles
+   * @returns {Promise<void>} Resolves once the condition holds
+   */
+  private waitFor;
+  /**
+   * Wakes everyone whose condition now holds.
+   *
+   * @returns {void}
+   */
+  private notify;
+}
+//#endregion