@c9up/bay 0.1.13 → 0.2.0

package/src/drivers/RedisDriver.ts

@@ -18,8 +18,9 @@
  * needs a thin adapter.
  */
 
+import { DEFAULT_QUEUE } from "../Job.js";
 import { inProduction } from "../nodeEnv.js";
-import type { Job, QueueDriver } from "../QueueManager.js";
+import { type JobRecord, type QueueDriver, queueOf } from "../QueueManager.js";
 
 export interface RedisClient {
 	rpush(key: string, ...values: string[]): Promise<number>;
@@ -31,25 +32,80 @@ export interface RedisClient {
 		to: "LEFT" | "RIGHT",
 	): Promise<string | null>;
 	lrem(key: string, count: number, element: string): Promise<number>;
+	/**
+	 * Optional, like `lmove`. Present on ioredis; without it the failed list
+	 * simply keeps its entries, which is what this driver did before.
+	 */
+	ltrim?(key: string, start: number, stop: number): Promise<string>;
 	llen(key: string): Promise<number>;
 	lrange(key: string, start: number, stop: number): Promise<string[]>;
 	del(key: string): Promise<number>;
 	set(key: string, value: string, ...args: string[]): Promise<string | null>;
 	get(key: string): Promise<string | null>;
+	/**
+	 * Sorted-set commands, for delayed jobs. Optional like `lmove`: a client
+	 * without them can still run a queue, and `push` refuses a job carrying a
+	 * `delay` rather than running it early — which is the one thing a delay
+	 * must not do.
+	 */
+	zadd?(key: string, score: number, member: string): Promise<number | string>;
+	zrangebyscore?(
+		key: string,
+		min: number | string,
+		max: number | string,
+		...args: string[]
+	): Promise<string[]>;
+	zrem?(key: string, ...members: string[]): Promise<number>;
+	zcard?(key: string): Promise<number>;
 }
 
-function isValidJob(obj: unknown): obj is Job {
+function isValidJob(obj: unknown): obj is JobRecord {
 	if (typeof obj !== "object" || obj === null) return false;
-	const j = obj as Record<string, unknown>;
 	return (
-		typeof j.id === "string" &&
-		typeof j.name === "string" &&
-		typeof j.attempts === "number" &&
-		typeof j.maxAttempts === "number" &&
-		typeof j.status === "string"
+		typeof Reflect.get(obj, "id") === "string" &&
+		typeof Reflect.get(obj, "name") === "string" &&
+		typeof Reflect.get(obj, "attempts") === "number" &&
+		typeof Reflect.get(obj, "maxAttempts") === "number" &&
+		typeof Reflect.get(obj, "status") === "string"
 	);
 }
 
+/**
+ * What a lease holds: the exact string pop() moved into `processing`, and the
+ * worker that moved it.
+ *
+ * The owner is what makes renewal safe. Without it a worker whose lease had
+ * already expired — its job recovered, re-popped by somebody else — would go on
+ * extending the deadline of a job it no longer had any claim on. Upstream draws
+ * the same line inside its renewal script: "Only the worker that currently owns
+ * the lease may renew it."
+ */
+interface Lease {
+	owner: string;
+	raw: string;
+}
+
+/**
+ * Read a lease back. A value written by an older version of this driver is the
+ * raw job string on its own, with no owner — still usable for the one thing
+ * `#removeFromProcessing` needs it for.
+ */
+function readLease(stored: string): { owner?: string; raw: string } {
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(stored);
+	} catch {
+		return { raw: stored };
+	}
+	if (typeof parsed !== "object" || parsed === null) return { raw: stored };
+	const raw = Reflect.get(parsed, "raw");
+	const owner = Reflect.get(parsed, "owner");
+	if (typeof raw !== "string" || typeof owner !== "string") {
+		return { raw: stored };
+	}
+	return { owner, raw };
+}
+
 /**
  * Where the client comes from. A resolver is what lets a queue name its
  * connection (`quasarConnection("jobs")`) instead of being handed a client:
@@ -157,6 +213,23 @@ export class RedisDriver implements QueueDriver {
 			 * deployment makes, not one a version check makes for it.
 			 */
 			allowNonAtomicPop?: boolean;
+			/**
+			 * How many times a job may be reclaimed from a stalled worker before
+			 * it is filed as failed instead of pushed round again. Default `1`,
+			 * upstream's default for the same setting.
+			 *
+			 * Unbounded recovery is a job that kills its worker taking the whole
+			 * queue down with it, forever: the crash never reaches the failure
+			 * path, so `attempts` never moves and `maxAttempts` never applies.
+			 */
+			maxStalledCount?: number;
+			/**
+			 * How many failed jobs to keep. Default `1000` — the ceiling the
+			 * memory driver already had. `0` keeps every one of them.
+			 *
+			 * Only enforced when the client answers `ltrim`.
+			 */
+			maxFailedJobs?: number;
 		},
 	) {
 		this.#source = source;
@@ -183,33 +256,91 @@ export class RedisDriver implements QueueDriver {
 			);
 		}
 		this.#visibilityTimeout = visibilityTimeout;
+
+		const maxStalled = options?.maxStalledCount ?? 1;
+		if (!Number.isInteger(maxStalled) || maxStalled < 0) {
+			throw new Error(
+				`[bay] RedisDriver maxStalledCount must be a non-negative integer, got ${maxStalled}`,
+			);
+		}
+		this.#maxStalledCount = maxStalled;
+
+		const maxFailed = options?.maxFailedJobs ?? 1000;
+		if (!Number.isInteger(maxFailed) || maxFailed < 0) {
+			throw new Error(
+				`[bay] RedisDriver maxFailedJobs must be a non-negative integer, got ${maxFailed}`,
+			);
+		}
+		this.#maxFailedJobs = maxFailed;
+	}
+
+	/**
+	 * Renew a lease at half its length: two chances to be heard before the
+	 * deadline, so one slow round-trip does not hand a running job to somebody
+	 * else. Read by `QueueManager` while a handler runs.
+	 */
+	get renewIntervalMs(): number {
+		return Math.max(1, Math.floor(this.#visibilityTimeout / 2));
 	}
 
-	#pendingKey = () => `${this.#prefix}pending`;
+	/**
+	 * Where one queue's jobs wait.
+	 *
+	 * The default queue keeps the key it always had. Naming it
+	 * `queue:default:pending` would have been tidier and would have orphaned
+	 * every job already sitting in `queue:pending` at the moment of the upgrade
+	 * — a silent loss, since nothing reads the old key afterwards.
+	 */
+	#pendingKey = (queue: string = DEFAULT_QUEUE) =>
+		queue === DEFAULT_QUEUE
+			? `${this.#prefix}pending`
+			: `${this.#prefix}q:${queue}:pending`;
+	#delayedKey = (queue: string = DEFAULT_QUEUE) =>
+		queue === DEFAULT_QUEUE
+			? `${this.#prefix}delayed`
+			: `${this.#prefix}q:${queue}:delayed`;
 	#processingKey = () => `${this.#prefix}processing`;
 	#allowNonAtomicPop = false;
+	#maxStalledCount = 1;
+	#maxFailedJobs = 1000;
+	/** This driver instance, as a lease owner. */
+	#workerId = crypto.randomUUID();
 	#failedKey = () => `${this.#prefix}failed`;
 	#leaseKey = (jobId: string) => `${this.#prefix}lease:${jobId}`;
 
-	async push(job: Job): Promise<void> {
+	async push(job: JobRecord): Promise<void> {
 		const client = await this.#client();
-		await client.rpush(this.#pendingKey(), JSON.stringify(job));
+		const queue = queueOf(job);
+		if (job.runAt !== undefined && job.runAt > Date.now()) {
+			if (!client.zadd) {
+				// Pushing it to the list instead would run it now, which is the
+				// one thing a delay exists to prevent.
+				throw new Error(
+					"This Redis client cannot hold a delayed job: it has no ZADD. " +
+						"Use a client with sorted-set commands (ioredis has them), or dispatch without `delay`.",
+				);
+			}
+			await client.zadd(
+				this.#delayedKey(queue),
+				job.runAt,
+				JSON.stringify(job),
+			);
+			return;
+		}
+		await client.rpush(this.#pendingKey(queue), JSON.stringify(job));
 	}
 
-	async pop(): Promise<Job | null> {
+	async pop(
+		queues: readonly string[] = [DEFAULT_QUEUE],
+	): Promise<JobRecord | null> {
 		const client = await this.#client();
 		let raw: string | null = null;
 
-		if (client.lmove) {
-			raw = await client.lmove(
-				this.#pendingKey(),
-				this.#processingKey(),
-				"LEFT",
-				"RIGHT",
-			);
-		} else {
-			raw = await client.lpop(this.#pendingKey());
-			if (raw) await client.rpush(this.#processingKey(), raw);
+		// In the order given, so a worker can say which queue it drains first.
+		for (const queue of queues) {
+			await this.#promoteDue(client, queue);
+			raw = await this.#take(client, queue);
+			if (raw !== null) break;
 		}
 
 		if (!raw) return null;
@@ -239,34 +370,54 @@ export class RedisDriver implements QueueDriver {
 		// pending — so the delivery guarantee survives the blip.
 		await client.set(
 			this.#leaseKey(parsed.id),
-			raw,
+			JSON.stringify({ owner: this.#workerId, raw } satisfies Lease),
 			"PX",
 			String(this.#visibilityTimeout),
 		);
 		return parsed;
 	}
 
-	async complete(job: Job): Promise<void> {
+	/**
+	 * Say the job is still being worked on, and push its deadline back.
+	 *
+	 * Answers `false` when there is nothing left to renew — the lease expired
+	 * and the job was recovered, or it was recovered and re-popped by another
+	 * worker, whose claim this one must not extend.
+	 */
+	async renew(job: JobRecord): Promise<boolean> {
+		const client = await this.#client();
+		const key = this.#leaseKey(job.id);
+		const stored = await client.get(key);
+		if (stored === null) return false;
+		const lease = readLease(stored);
+		if (lease.owner !== undefined && lease.owner !== this.#workerId) {
+			return false;
+		}
+		await client.set(key, stored, "PX", String(this.#visibilityTimeout));
+		return true;
+	}
+
+	async complete(job: JobRecord): Promise<void> {
 		const client = await this.#client();
 		await this.#removeFromProcessing(job);
 		await client.del(this.#leaseKey(job.id));
 	}
 
-	async fail(job: Job, error: string): Promise<void> {
+	async fail(job: JobRecord, error: string): Promise<void> {
 		const client = await this.#client();
 		await this.#removeFromProcessing(job);
 		await client.del(this.#leaseKey(job.id));
 		job.error = error;
 		job.status = "failed";
-		await client.rpush(this.#failedKey(), JSON.stringify(job));
+		await this.#pushFailed(client, job);
 	}
 
-	async retry(job: Job): Promise<void> {
+	async retry(job: JobRecord): Promise<void> {
 		const client = await this.#client();
 		await this.#removeFromProcessing(job);
 		await client.del(this.#leaseKey(job.id));
 		job.status = "pending";
-		await client.rpush(this.#pendingKey(), JSON.stringify(job));
+		await client.rpush(this.#pendingKey(queueOf(job)), JSON.stringify(job));
 	}
 
 	async recoverStale(): Promise<number> {
@@ -288,30 +439,77 @@ export class RedisDriver implements QueueDriver {
 				continue;
 			}
 			const lease = await client.get(this.#leaseKey(parsed.id));
-			if (lease === null) {
-				await client.lrem(this.#processingKey(), 1, raw);
-				parsed.status = "pending";
-				await client.rpush(this.#pendingKey(), JSON.stringify(parsed));
-				recovered++;
+			if (lease !== null) continue;
+
+			// The LREM is the claim, and its RESULT decides who acts. Two
+			// recovery passes overlapping — two workers, or one worker whose
+			// pass ran long — both read the same expired entry, and both used to
+			// push it back to pending: one job, delivered twice, from the
+			// mechanism that exists to make delivery reliable. Exactly one LREM
+			// can remove a given element, so exactly one pass continues past
+			// here. (A crash between this and the RPUSH below still loses the
+			// entry; closing that needs the whole pass in one server-side script,
+			// which is how upstream does it.)
+			const claimed = await client.lrem(this.#processingKey(), 1, raw);
+			if (claimed === 0) continue;
+
+			// A stall is not an attempt: the worker died before the handler
+			// could fail, so `attempts` never moved and `maxAttempts` never
+			// applied. A job that kills whatever picks it up was therefore
+			// recovered forever, taking the queue with it. Counted separately,
+			// and bounded — upstream bounds the same thing with the same
+			// default, failing the job once it is exceeded.
+			const stalled = (parsed.stalledCount ?? 0) + 1;
+			if (stalled > this.#maxStalledCount) {
+				parsed.stalledCount = stalled;
+				parsed.status = "failed";
+				parsed.error = `Stalled ${stalled} time(s) without completing (maxStalledCount ${this.#maxStalledCount})`;
+				await this.#pushFailed(client, parsed);
+				continue;
 			}
+
+			parsed.stalledCount = stalled;
+			parsed.status = "pending";
+			// Back to the queue it came from, not to the default one: a recovered
+			// job whose queue nobody serves would never run again.
+			await client.rpush(
+				this.#pendingKey(queueOf(parsed)),
+				JSON.stringify(parsed),
+			);
+			recovered++;
 		}
 		return recovered;
 	}
 
+	/**
+	 * File a job as failed, keeping the list to `maxFailedJobs`.
+	 *
+	 * Unbounded, the failed list is a leak with no ceiling and no owner: nothing
+	 * trims it, and `failed()` reads all of it in one LRANGE. The memory driver
+	 * has capped its own at a thousand from the start; this is the same cap on
+	 * the driver where the list actually survives a restart.
+	 */
+	async #pushFailed(client: RedisClient, job: JobRecord): Promise<void> {
+		await client.rpush(this.#failedKey(), JSON.stringify(job));
+		if (this.#maxFailedJobs === 0 || !client.ltrim) return;
+		await client.ltrim(this.#failedKey(), -this.#maxFailedJobs, -1);
+	}
+
 	/**
 	 * Remove the entry for `job` from the processing list. The string in
 	 * Redis is whatever pop() pushed, but QueueManager mutates `job` after
 	 * pop returns (attempts++, status="processing", processedAt, then
 	 * completed/failed/pending). LREM-ing on `JSON.stringify(job)` would
-	 * therefore miss every real-world entry. Use the lease — set to the
-	 * exact raw string at pop() time — and fall back to a list scan when
+	 * therefore miss every real-world entry. Use the lease — which carries
+	 * the exact raw string pop() moved — and fall back to a list scan when
 	 * the lease has expired (e.g. recoverStale already handled it).
 	 */
-	async #removeFromProcessing(job: Job): Promise<void> {
+	async #removeFromProcessing(job: JobRecord): Promise<void> {
 		const client = await this.#client();
 		const stored = await client.get(this.#leaseKey(job.id));
 		if (stored !== null) {
-			const removed = await client.lrem(this.#processingKey(), 1, stored);
+			const { raw } = readLease(stored);
+			const removed = await client.lrem(this.#processingKey(), 1, raw);
 			if (removed > 0) return;
 		}
 		// Lease missing or already-LREM'd entry not found — best-effort scan
@@ -324,14 +522,14 @@ export class RedisDriver implements QueueDriver {
 			} catch {
 				continue;
 			}
-			if (isValidJob(parsed) && (parsed as { id: string }).id === job.id) {
+			if (isValidJob(parsed) && parsed.id === job.id) {
 				await client.lrem(this.#processingKey(), 1, item);
 				return;
 			}
 		}
 	}
 
-	async failed(): Promise<Job[]> {
+	async failed(): Promise<JobRecord[]> {
 		const client = await this.#client();
 		const raws = await client.lrange(this.#failedKey(), 0, -1);
 		return raws
@@ -343,11 +541,69 @@ export class RedisDriver implements QueueDriver {
 					return null;
 				}
 			})
-			.filter((j): j is Job => j !== null);
+			.filter((j): j is JobRecord => j !== null);
 	}
 
-	async size(): Promise<number> {
+	/**
+	 * How many jobs are waiting on `queue` — its list plus its delayed set.
+	 *
+	 * A delayed job is queued; it is simply not due. Counting only the list
+	 * reported an empty queue to anything draining one before shutdown.
+	 */
+	async size(queue: string = DEFAULT_QUEUE): Promise<number> {
 		const client = await this.#client();
-		return client.llen(this.#pendingKey());
+		const waiting = await client.llen(this.#pendingKey(queue));
+		const delayed = client.zcard
+			? await client.zcard(this.#delayedKey(queue))
+			: 0;
+		return waiting + delayed;
+	}
+
+	/**
+	 * Move `queue`'s due jobs out of the delayed set and onto its list.
+	 *
+	 * `ZREM` is the claim: two workers can read the same due entry, and only
+	 * the one whose removal returns 1 owns it. Without that the job is pushed
+	 * onto the list once per worker that saw it.
+	 */
+	async #promoteDue(client: RedisClient, queue: string): Promise<void> {
+		if (!client.zrangebyscore || !client.zrem) return;
+		const due = await client.zrangebyscore(
+			this.#delayedKey(queue),
+			0,
+			Date.now(),
+			"LIMIT",
+			"0",
+			"100",
+		);
+		for (const raw of due) {
+			const claimed = await client.zrem(this.#delayedKey(queue), raw);
+			if (claimed !== 1) continue;
+			let parsed: unknown;
+			try {
+				parsed = JSON.parse(raw);
+			} catch {
+				// Already removed from the set; there is nothing runnable to push.
+				continue;
+			}
+			if (!isValidJob(parsed)) continue;
+			parsed.runAt = undefined;
+			await client.rpush(this.#pendingKey(queue), JSON.stringify(parsed));
+		}
+	}
+
+	/** Take the head of one queue, claiming it in `processing`. */
+	async #take(client: RedisClient, queue: string): Promise<string | null> {
+		if (client.lmove) {
+			return client.lmove(
+				this.#pendingKey(queue),
+				this.#processingKey(),
+				"LEFT",
+				"RIGHT",
+			);
+		}
+		const raw = await client.lpop(this.#pendingKey(queue));
+		if (raw) await client.rpush(this.#processingKey(), raw);
+		return raw;
 	}
 }

package/src/index.ts

@@ -1,18 +1,47 @@
 /**
  * @c9up/bay — Background job queue for the Ream framework.
  *
- * Dispatch/process/retry/fail pattern with pluggable drivers (Memory, Redis).
+ * A job is a class: it carries its own name, its options and the type of the
+ * payload it reads, and `dispatch(SomeJob, payload)` takes it. Named queues,
+ * `delay`, `timeout` and worker `concurrency` come from the same declaration.
  *
- * @implements MISS-11
+ * Dispatch/process/retry/fail, with pluggable drivers (Memory, Redis).
  */
 
-export type { BayProviderConfig } from "./BayProvider.js";
+export {
+	type AdapterFactory,
+	drivers,
+	type QueueStoreFactory,
+	stores,
+} from "./adapters.js";
+
+import "./augmentations.js";
+
+export type {
+	BayAppContext,
+	BayConfigStore,
+	BayContainer,
+	BayProviderConfig,
+} from "./BayProvider.js";
 export { MemoryDriver } from "./drivers/MemoryDriver.js";
 export type { RedisClient } from "./drivers/RedisDriver.js";
 export { RedisDriver } from "./drivers/RedisDriver.js";
-export type { Job, JobHandler, QueueDriver } from "./QueueManager.js";
-export { QueueManager } from "./QueueManager.js";
-export { type QueueStoreFactory, stores } from "./stores.js";
+export {
+	DEFAULT_QUEUE,
+	type Duration,
+	isJobClass,
+	Job,
+	type JobClass,
+	type JobOptions,
+} from "./Job.js";
+export type {
+	DispatchOptions,
+	JobHandler,
+	JobRecord,
+	QueueDriver,
+	WorkerOptions,
+} from "./QueueManager.js";
+export { QueueManager, queueOf } from "./QueueManager.js";
 
 import type { BayProviderConfig } from "./BayProvider.js";
 

package/src/jobs.ts

@@ -0,0 +1,111 @@
+/**
+ * Finding the job classes an application wrote.
+ *
+ * A job class carries its own name, and a worker resolves a queued record by
+ * that name — so the worker process has to have imported the class. Written by
+ * hand that is a registration list to keep in step with a directory:
+ *
+ *     queue.registerJob(SendEmail)
+ *     queue.registerJob(SendInvoice)   // …and the one nobody added
+ *
+ * `locations` in `config/queue.ts` is the directory instead. Every module under
+ * it is imported once at boot, and a default export that is a job class is
+ * registered under its own name.
+ *
+ * Directories, not globs. `'./app/jobs/**\/*.{ts,js}'` — the spelling upstream's
+ * config uses — is accepted and read as the directory it starts with, so a
+ * config copied from there works; bay ships no glob engine and adding a
+ * dependency for one path shape is not worth it.
+ */
+
+import * as fsp from "node:fs/promises";
+import * as path from "node:path";
+import { pathToFileURL } from "node:url";
+import { isJobClass, type JobClass } from "./Job.js";
+
+/** Where `make:job` writes, and where discovery looks when nothing is declared. */
+export const DEFAULT_JOBS_DIR = "app/jobs";
+
+let jobsDir = DEFAULT_JOBS_DIR;
+
+/** @internal Told by the provider what the config declared. */
+export function setJobsDir(dir: string): void {
+	jobsDir = dir;
+}
+
+/** Where job files live — the first `locations` entry, or the default. */
+export function getJobsDir(): string {
+	return jobsDir;
+}
+
+/**
+ * The directory a `locations` entry names.
+ *
+ * Everything from the first glob character on is dropped: `app/jobs/** /*.ts`
+ * and `app/jobs` name the same directory, and the walk below is recursive
+ * either way.
+ */
+export function directoryOf(location: string): string {
+	const withoutGlob = location.split(/[*?[{]/)[0] ?? location;
+	const trimmed = withoutGlob.replace(/\/+$/, "");
+	return trimmed.replace(/^\.\//, "") || ".";
+}
+
+/** Every module file under `dir`, recursively. */
+async function walk(dir: string, depth = 0): Promise<string[]> {
+	// A jobs directory is a flat convention with the occasional subdirectory;
+	// an unbounded walk would follow whatever happens to live under it.
+	if (depth > 8) return [];
+	let entries: import("node:fs").Dirent[];
+	try {
+		entries = await fsp.readdir(dir, { withFileTypes: true });
+	} catch {
+		// A declared directory that does not exist yet is not an error: a project
+		// can name where its jobs will go before writing the first one.
+		return [];
+	}
+	const found: string[] = [];
+	for (const entry of entries) {
+		const full = path.join(dir, entry.name);
+		if (entry.isDirectory()) {
+			found.push(...(await walk(full, depth + 1)));
+			continue;
+		}
+		// `.d.ts` is a declaration, not a module with a job in it.
+		if (/\.d\.[cm]?ts$/.test(entry.name)) continue;
+		if (/\.[cm]?[jt]s$/.test(entry.name)) found.push(full);
+	}
+	return found.sort();
+}
+
+/**
+ * Import every module under `locations` and return the job classes they
+ * default-export.
+ *
+ * A module that throws on import is reported and skipped: one unfinished job
+ * file must not stop the worker from running every other job.
+ */
+export async function discoverJobs(
+	locations: readonly string[],
+): Promise<JobClass[]> {
+	const found: JobClass[] = [];
+	for (const location of locations) {
+		for (const file of await walk(directoryOf(location))) {
+			let module: unknown;
+			try {
+				module = await import(pathToFileURL(path.resolve(file)).href);
+			} catch (err) {
+				process.stderr.write(
+					`[bay] could not load '${file}': ${
+						err instanceof Error ? err.message : String(err)
+					}\n`,
+				);
+				continue;
+			}
+			if (typeof module !== "object" || module === null) continue;
+			const exported = Reflect.get(module, "default");
+			if (isJobClass(exported)) found.push(exported);
+		}
+	}
+	return found;
+}

package/src/quasar.ts

@@ -3,7 +3,7 @@
  *
  * Bay does not depend on quasar: it is an optional peer, and this module
  * never imports it statically. The specifier is built at runtime so the
- * TypeScript build stays free of it too — a hard type import would make echo
+ * TypeScript build stays free of it too — a hard type import would make bay
  * unbuildable for anyone who queues in memory.
  *
  * The shape is checked before use rather than asserted, the same way bay