@arkstack/queue 0.19.3 → 0.19.5

package/README.md

@@ -49,12 +49,29 @@ await Queue.clear('emails')
 ## Workers
 
 ```bash
-ark queue:work                      # daemon on the default connection
+ark queue:work                      # daemon on the default connection's default queue
 ark queue:work redis --queue=emails
+ark queue:work --queue=high,default # priority list: high first, then default
+ark queue:work --queue=*            # every queue on the connection
 ark queue:work --once               # process a single job and exit
 ark queue:clear redis --queue=emails
 ```
 
+`--queue` takes one queue, a comma-separated list, or `*`:
+
+- **No `--queue`**: only the connection's default queue (`queue` in its config). Jobs
+  sent elsewhere with `onQueue(...)` or a job's `queue` property are **not** processed.
+- **A list** (`--queue=webhook,ocr,default`) is a strict priority order: on every
+  loop the worker takes the first job it finds walking the list, so later queues
+  run only while earlier ones are empty.
+- **`*`** works every queue on the connection. The `database` driver takes the
+  oldest due job across all queues; the `redis` driver discovers queues with
+  `SCAN` and works them round-robin. Combine it with names to prioritize some:
+  `--queue=webhook,*`.
+
+Every queue that receives jobs needs a worker that lists it (or `*`), or its jobs
+wait forever.
+
 Programmatically:
 
 ```ts
@@ -62,8 +79,12 @@ import { Queue } from '@arkstack/queue'
 
 const worker = Queue.worker('database')
 await worker.daemon({ queue: 'default', sleep: 3 })
+await worker.daemon({ queue: ['webhook', '*'], sleep: 3 }) // priority list / wildcard
 ```
 
+A custom driver supports `*` by overriding `popAny()` on `QueueContract`; the base
+implementation works only the default queue.
+
 A job is **deleted** on success, **released** for another attempt on failure, and **marked failed** (invoking its `failed` hook) once it exhausts `tries`.
 
 ## Job (de)serialization

package/dist/{QueueManager--4wZ8gJz.js → QueueManager-HqKDfrUq.js}

@@ -152,6 +152,15 @@ var QueueContract = class {
 	getDefaultQueue() {
 		return "default";
 	}
+	/**
+	* Reserve the next job from any queue on this connection, for the worker's `*`
+	* wildcard. Drivers that can list their queues override this; the default
+	* works only the default queue, so a driver without that ability behaves
+	* exactly as a worker given no queue.
+	*/
+	async popAny() {
+		return this.pop();
+	}
 };
 //#endregion
 //#region src/drivers/DatabaseQueue.ts
@@ -210,9 +219,25 @@ var DatabaseQueue = class extends QueueContract {
 	}
 	async pop(queue) {
 		const db = await this.database();
-		const name = queue ?? this.defaultQueue;
-		const row = await this.availableQuery(db, name).orderBy({ id: "asc" }).first();
-		if (!row) return null;
+		const row = await this.availableQuery(db, queue ?? this.defaultQueue).orderBy({ id: "asc" }).first();
+		return row ? this.reserve(db, row) : null;
+	}
+	/**
+	* The oldest due job on any queue. Ordering by id across queues is FIFO by
+	* arrival, so a busy queue can't starve a quiet one.
+	*/
+	async popAny() {
+		const db = await this.database();
+		const row = await this.availableQuery(db).orderBy({ id: "asc" }).first();
+		return row ? this.reserve(db, row) : null;
+	}
+	/**
+	* Reserve a fetched row and wrap it as a {@link Job}.
+	*
+	* @param db
+	* @param row
+	*/
+	async reserve(db, row) {
 		const attempts = row.attempts + 1;
 		await db.table(this.options.table).where({ id: row.id }).update({
 			reserved_at: now$1(),
@@ -221,7 +246,7 @@ var DatabaseQueue = class extends QueueContract {
 		return new Job({
 			...JSON.parse(row.payload),
 			attempts
-		}, name, {
+		}, row.queue, {
 			delete: async () => {
 				await db.table(this.options.table).where({ id: row.id }).delete();
 			},
@@ -244,15 +269,17 @@ var DatabaseQueue = class extends QueueContract {
 		return count;
 	}
 	/**
-	* Due, unreserved jobs on the given queue, oldest first. Released jobs become
-	* unreserved with a future `available_at`, so they naturally wait their turn.
+	* Due, unreserved jobs, oldest first: on one queue, or on every queue when
+	* `queue` is omitted. Released jobs become unreserved with a future
+	* `available_at`, so they naturally wait their turn.
 	*
 	* @param db
 	* @param queue
 	* @returns
 	*/
 	availableQuery(db, queue) {
-		return db.table(this.options.table).where({ queue }).whereNull("reserved_at").whereRaw("available_at <= " + now$1());
+		const table = db.table(this.options.table);
+		return (queue === void 0 ? table : table.where({ queue })).whereNull("reserved_at").whereRaw("available_at <= " + now$1());
 	}
 };
 //#endregion
@@ -269,6 +296,8 @@ const now = () => Math.floor(Date.now() / 1e3);
 var RedisQueue = class extends QueueContract {
 	options;
 	client;
+	/** Rotates which queue {@link popAny} tries first, so none is starved. */
+	anyCursor = 0;
 	constructor(options) {
 		super();
 		this.options = options;
@@ -342,6 +371,22 @@ var RedisQueue = class extends QueueContract {
 			}
 		});
 	}
+	/**
+	* Reserve the next job from any queue. Redis has no ordering across lists, so
+	* the queues are discovered with `SCAN` (never the blocking `KEYS`) and tried
+	* round-robin, starting one further along on each call.
+	*/
+	async popAny() {
+		const queues = await this.queueNames(await this.connection());
+		for (let offset = 0; offset < queues.length; offset++) {
+			const job = await this.pop(queues[(this.anyCursor + offset) % queues.length]);
+			if (job) {
+				this.anyCursor = (this.anyCursor + offset + 1) % queues.length;
+				return job;
+			}
+		}
+		return null;
+	}
 	async size(queue) {
 		const client = await this.connection();
 		const name = queue ?? this.defaultQueue;
@@ -360,6 +405,21 @@ var RedisQueue = class extends QueueContract {
 		return count;
 	}
 	/**
+	* Every queue with jobs in any state (waiting, delayed or reserved), sorted so
+	* the round-robin order is stable.
+	*/
+	async queueNames(client) {
+		const prefix = this.options.prefix ?? "queues:";
+		const names = /* @__PURE__ */ new Set();
+		let cursor = "0";
+		do {
+			const [next, keys] = await client.scan(cursor, "MATCH", `${prefix}*`, "COUNT", 100);
+			for (const key of keys) names.add(key.slice(prefix.length).replace(/:(delayed|reserved)$/, ""));
+			cursor = next;
+		} while (cursor !== "0");
+		return [...names].sort();
+	}
+	/**
 	* Move due delayed jobs and expired reservations back onto the waiting list.
 	*/
 	async migrate(client, queue) {
@@ -419,6 +479,8 @@ var SyncQueue = class extends QueueContract {
 };
 //#endregion
 //#region src/Worker.ts
+/** Matches every queue on the connection in a worker's queue list. */
+const ANY_QUEUE = "*";
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /**
 * Pulls jobs from a connection and executes them.
@@ -450,15 +512,31 @@ var Worker = class {
 	}
 	/**
 	* Pop and process the next job. Returns `true` if a job was processed,
-	* `false` when the queue was empty.
+	* `false` when every queue asked for was empty.
+	*
+	* @param queue  A queue, or a priority-ordered list of queues (see
+	*               {@link WorkerOptions.queue}). Omitted: the connection's default.
 	*/
 	async runNextJob(queue) {
-		const job = await this.connection.pop(queue);
+		const job = await this.nextJob(queue);
 		if (!job) return false;
 		await this.process(job);
 		return true;
 	}
 	/**
+	* Reserve the next job, walking the queues in priority order.
+	*
+	* @param queue
+	*/
+	async nextJob(queue) {
+		const queues = Array.isArray(queue) ? queue : [queue];
+		for (const name of queues) {
+			const job = name === "*" ? await this.connection.popAny() : await this.connection.pop(name);
+			if (job) return job;
+		}
+		return null;
+	}
+	/**
 	* Execute a single reserved job, handling success and failure.
 	*/
 	async process(job) {
@@ -631,4 +709,4 @@ var Queue = class {
 	}
 };
 //#endregion
-export { RedisQueue as a, Job as c, serializeJob as d, setResolver as f, SyncQueue as i, resetSerialization as l, configure as n, DatabaseQueue as o, setSerializer as p, Worker as r, QueueContract as s, Queue as t, resolveJob as u };
+export { SyncQueue as a, QueueContract as c, resolveJob as d, serializeJob as f, Worker as i, Job as l, setSerializer as m, configure as n, RedisQueue as o, setResolver as p, ANY_QUEUE as r, DatabaseQueue as s, Queue as t, resetSerialization as u };

package/dist/commands/QueueClearCommand.js

@@ -1,4 +1,4 @@
-import { t as Queue } from "../QueueManager--4wZ8gJz.js";
+import { t as Queue } from "../QueueManager-HqKDfrUq.js";
 import { Command } from "@h3ravel/musket";
 import { bootArkorm } from "@arkstack/database";
 //#region src/commands/QueueClearCommand.ts

package/dist/commands/QueueWorkCommand.d.ts

@@ -7,6 +7,19 @@ declare class QueueWorkCommand extends Command {
   protected signature: string;
   protected description: string;
   handle(): Promise<void>;
+  /**
+   * The queues to work, from `--queue`: `ocr` → `['ocr']`, `webhook,ocr` → a
+   * priority list, `*` → every queue. Falls back to the connection's default.
+   *
+   * @param fallback  The connection's default queue.
+   */
+  private queues;
+  /**
+   * How the worker's queues read in its startup line.
+   *
+   * @param queues
+   */
+  private describe;
   /**
    * Read a multi-word flag. Musket hands over the parsed options camelCased,
    * so `--stop-when-empty` arrives as `stopWhenEmpty`; the kebab-case name is

package/dist/commands/QueueWorkCommand.js

@@ -1,4 +1,4 @@
-import { r as Worker, t as Queue } from "../QueueManager--4wZ8gJz.js";
+import { i as Worker, t as Queue } from "../QueueManager-HqKDfrUq.js";
 import { Command } from "@h3ravel/musket";
 import { bootArkorm } from "@arkstack/database";
 //#region src/commands/QueueWorkCommand.ts
@@ -10,7 +10,7 @@ const reason = (error) => error instanceof Error ? error.message : String(error)
 var QueueWorkCommand = class extends Command {
 	signature = `queue:work
         {connection? : The queue connection to work. Defaults to the configured default.}
-        {--queue= : The queue to process.}
+        {--queue= : The queue(s) to process: a name, a comma-separated priority list, or * for every queue.}
         {--sleep=3 : Seconds to sleep when no job is available.}
         {--max-jobs=0 : Stop after processing this many jobs (0 for unlimited).}
         {--once : Process a single job and exit.}
@@ -27,7 +27,7 @@ var QueueWorkCommand = class extends Command {
 				this.error(released ? `Failed (attempt ${job.attempts()}, retrying): ${job.name()} — ${reason(error)}` : `Failed permanently: ${job.name()} — ${reason(error)}`);
 			}
 		});
-		const queue = this.option("queue") ?? connection.getDefaultQueue();
+		const queue = this.queues(connection.getDefaultQueue());
 		try {
 			bootArkorm();
 		} catch {}
@@ -36,7 +36,7 @@ var QueueWorkCommand = class extends Command {
 			if (!await worker.runNextJob(queue)) this.info("No jobs available.");
 			return;
 		}
-		this.info(`Processing jobs from the [${connection.getConnectionName()}] connection on the [${queue}] queue.`);
+		this.info(`Processing jobs from the [${connection.getConnectionName()}] connection on ${this.describe(queue)}.`);
 		await worker.daemon({
 			queue,
 			sleep: Number(this.option("sleep") ?? 3),
@@ -45,6 +45,25 @@ var QueueWorkCommand = class extends Command {
 		});
 	}
 	/**
+	* The queues to work, from `--queue`: `ocr` → `['ocr']`, `webhook,ocr` → a
+	* priority list, `*` → every queue. Falls back to the connection's default.
+	*
+	* @param fallback  The connection's default queue.
+	*/
+	queues(fallback) {
+		const names = String(this.option("queue") ?? "").split(",").map((name) => name.trim()).filter(Boolean);
+		return names.length ? [...new Set(names)] : [fallback];
+	}
+	/**
+	* How the worker's queues read in its startup line.
+	*
+	* @param queues
+	*/
+	describe(queues) {
+		if (queues.length === 1) return queues[0] === "*" ? "every queue" : `the [${queues[0]}] queue`;
+		return `the [${queues.join(", ")}] queues, in that order`;
+	}
+	/**
 	* Read a multi-word flag. Musket hands over the parsed options camelCased,
 	* so `--stop-when-empty` arrives as `stopWhenEmpty`; the kebab-case name is
 	* accepted too so the flag can't go quietly missing again.

package/dist/index.d.ts

@@ -88,6 +88,13 @@ declare abstract class QueueContract {
    * Reserve and return the next available job, or `null` when none is ready.
    */
   abstract pop(queue?: string): Promise<Job | null>;
+  /**
+   * Reserve the next job from any queue on this connection, for the worker's `*`
+   * wildcard. Drivers that can list their queues override this; the default
+   * works only the default queue, so a driver without that ability behaves
+   * exactly as a worker given no queue.
+   */
+  popAny(): Promise<Job | null>;
   /**
    * The number of jobs waiting on the given queue.
    */
@@ -232,13 +239,20 @@ interface RedisClient {
   zrem(key: string, member: string): Promise<number>;
   zcard(key: string): Promise<number>;
   zrangebyscore(key: string, min: string | number, max: string | number): Promise<string[]>;
+  scan(cursor: string | number, ...args: (string | number)[]): Promise<[cursor: string, keys: string[]]>;
   quit(): Promise<unknown>;
 }
 //#endregion
 //#region src/Worker.d.ts
+/** Matches every queue on the connection in a worker's queue list. */
+declare const ANY_QUEUE = "*";
 interface WorkerOptions {
-  /** The queue name to pull from. */
-  queue?: string;
+  /**
+   * The queue(s) to pull from. A list is a strict priority order: each loop
+   * takes the first job found walking the list. {@link ANY_QUEUE} (`*`) matches
+   * any queue, so `['webhook', '*']` means "webhook first, then anything".
+   */
+  queue?: string | string[];
   /** Seconds to wait when the queue is empty before polling again. */
   sleep?: number;
   /** Stop after processing this many jobs (0 = unlimited). */
@@ -285,9 +299,18 @@ declare class Worker {
   on(handlers: WorkerHandlers): this;
   /**
    * Pop and process the next job. Returns `true` if a job was processed,
-   * `false` when the queue was empty.
+   * `false` when every queue asked for was empty.
+   *
+   * @param queue  A queue, or a priority-ordered list of queues (see
+   *               {@link WorkerOptions.queue}). Omitted: the connection's default.
+   */
+  runNextJob(queue?: string | string[]): Promise<boolean>;
+  /**
+   * Reserve the next job, walking the queues in priority order.
+   *
+   * @param queue
    */
-  runNextJob(queue?: string): Promise<boolean>;
+  private nextJob;
   /**
    * Execute a single reserved job, handling success and failure.
    */
@@ -436,11 +459,24 @@ declare class DatabaseQueue extends QueueContract {
   pushRaw(payload: JobPayload, queue?: string, availableAt?: number): Promise<string>;
   later(delay: number | Date, job: Queueable, queue?: string): Promise<string>;
   pop(queue?: string): Promise<Job | null>;
+  /**
+   * The oldest due job on any queue. Ordering by id across queues is FIFO by
+   * arrival, so a busy queue can't starve a quiet one.
+   */
+  popAny(): Promise<Job | null>;
+  /**
+   * Reserve a fetched row and wrap it as a {@link Job}.
+   *
+   * @param db
+   * @param row
+   */
+  private reserve;
   size(queue?: string): Promise<number>;
   clear(queue?: string): Promise<number>;
   /**
-   * Due, unreserved jobs on the given queue, oldest first. Released jobs become
-   * unreserved with a future `available_at`, so they naturally wait their turn.
+   * Due, unreserved jobs, oldest first: on one queue, or on every queue when
+   * `queue` is omitted. Released jobs become unreserved with a future
+   * `available_at`, so they naturally wait their turn.
    *
    * @param db
    * @param queue
@@ -461,6 +497,8 @@ declare class DatabaseQueue extends QueueContract {
 declare class RedisQueue extends QueueContract {
   private readonly options;
   private client?;
+  /** Rotates which queue {@link popAny} tries first, so none is starved. */
+  private anyCursor;
   constructor(options: RedisConnectionConfig);
   private get defaultQueue();
   /** The configured queue this connection works when none is named. */
@@ -474,12 +512,23 @@ declare class RedisQueue extends QueueContract {
   pushRaw(payload: JobPayload, queue?: string, availableAt?: number): Promise<string>;
   later(delay: number | Date, job: Queueable, queue?: string): Promise<string>;
   pop(queue?: string): Promise<Job | null>;
+  /**
+   * Reserve the next job from any queue. Redis has no ordering across lists, so
+   * the queues are discovered with `SCAN` (never the blocking `KEYS`) and tried
+   * round-robin, starting one further along on each call.
+   */
+  popAny(): Promise<Job | null>;
   size(queue?: string): Promise<number>;
   clear(queue?: string): Promise<number>;
+  /**
+   * Every queue with jobs in any state (waiting, delayed or reserved), sorted so
+   * the round-robin order is stable.
+   */
+  private queueNames;
   /**
    * Move due delayed jobs and expired reservations back onto the waiting list.
    */
   private migrate;
 }
 //#endregion
-export { CustomQueueConnectionRegistry, DatabaseConnectionConfig, DatabaseFacade, DatabaseQueue, Job, JobHandlers, JobPayload, JobResolver, JobRow, JobSerializer, Query, Queue, QueueConfig, QueueConnectionConfig, QueueConnectionFactory, QueueContract, Queueable, RedisClient, RedisConnectionConfig, RedisQueue, SyncConnectionConfig, SyncQueue, Worker, WorkerHandlers, WorkerOptions, configure, resetSerialization, resolveJob, serializeJob, setResolver, setSerializer };
+export { ANY_QUEUE, CustomQueueConnectionRegistry, DatabaseConnectionConfig, DatabaseFacade, DatabaseQueue, Job, JobHandlers, JobPayload, JobResolver, JobRow, JobSerializer, Query, Queue, QueueConfig, QueueConnectionConfig, QueueConnectionFactory, QueueContract, Queueable, RedisClient, RedisConnectionConfig, RedisQueue, SyncConnectionConfig, SyncQueue, Worker, WorkerHandlers, WorkerOptions, configure, resetSerialization, resolveJob, serializeJob, setResolver, setSerializer };

package/dist/index.js

@@ -1,2 +1,2 @@
-import { a as RedisQueue, c as Job, d as serializeJob, f as setResolver, i as SyncQueue, l as resetSerialization, n as configure, o as DatabaseQueue, p as setSerializer, r as Worker, s as QueueContract, t as Queue, u as resolveJob } from "./QueueManager--4wZ8gJz.js";
-export { DatabaseQueue, Job, Queue, QueueContract, RedisQueue, SyncQueue, Worker, configure, resetSerialization, resolveJob, serializeJob, setResolver, setSerializer };
+import { a as SyncQueue, c as QueueContract, d as resolveJob, f as serializeJob, i as Worker, l as Job, m as setSerializer, n as configure, o as RedisQueue, p as setResolver, r as ANY_QUEUE, s as DatabaseQueue, t as Queue, u as resetSerialization } from "./QueueManager-HqKDfrUq.js";
+export { ANY_QUEUE, DatabaseQueue, Job, Queue, QueueContract, RedisQueue, SyncQueue, Worker, configure, resetSerialization, resolveJob, serializeJob, setResolver, setSerializer };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arkstack/queue",
-  "version": "0.19.3",
+  "version": "0.19.5",
   "type": "module",
   "description": "Queue module for Arkstack, providing a driver based queue transport and worker for background processing.",
   "homepage": "https://arkstack.toneflix.net/guide/queue",
@@ -34,13 +34,13 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@arkstack/common": "^0.19.3"
+    "@arkstack/common": "^0.19.5"
   },
   "peerDependencies": {
     "@h3ravel/musket": "^2.2.13",
     "ioredis": "^5.4.1",
-    "@arkstack/contract": "^0.19.3",
-    "@arkstack/database": "^0.19.3"
+    "@arkstack/database": "^0.19.5",
+    "@arkstack/contract": "^0.19.5"
   },
   "peerDependenciesMeta": {
     "@arkstack/database": {