@c9up/bay 0.1.12 → 0.1.13

package/src/drivers/RedisDriver.ts

@@ -18,6 +18,7 @@
  * needs a thin adapter.
  */
 
+import { inProduction } from "../nodeEnv.js";
 import type { Job, QueueDriver } from "../QueueManager.js";
 
 export interface RedisClient {
@@ -65,13 +66,35 @@ export type RedisClientSource =
  * has no client to inspect yet.
  */
 const warned = new WeakSet<object>();
-function warnWithoutLmove(client: RedisClient): void {
-	if (typeof client.lmove === "function" || warned.has(client)) return;
+function checkLmove(client: RedisClient, allowNonAtomicPop: boolean): void {
+	if (typeof client.lmove === "function") return;
+
+	// A queue's whole promise is that a job it accepted gets run. Without LMOVE
+	// the pop is `lpop` then `rpush`, and a crash between the two deletes the
+	// job from pending before it reaches processing: nothing recovers it,
+	// because nothing knows it existed. That is a different product, and in
+	// production it must be asked for rather than fallen into.
+	if (inProduction() && !allowNonAtomicPop) {
+		throw new Error(
+			"[bay] this Redis client has no LMOVE (Redis < 6.2), so pop() would be a non-atomic lpop+rpush — " +
+				"a crash between the two loses the in-flight job, turning at-least-once delivery into at-most-once.\n" +
+				"  Upgrade to Redis 6.2 or later, or pass `allowNonAtomicPop: true` to state that losing a job is acceptable here.",
+		);
+	}
+	if (warned.has(client)) return;
 	warned.add(client);
+
+	// Said even when the deployment opted in: agreeing to lose a job once, in a
+	// config file, is not the same as being reminded that this process is
+	// running that way. The line has to be in the logs of the incident.
+	const optedIn = inProduction() && allowNonAtomicPop;
 	console.warn(
 		"[bay] RedisDriver: client lacks LMOVE (Redis <6.2). pop() falls back to " +
 			"a non-atomic lpop+rpush, downgrading delivery from at-least-once to " +
-			"at-most-once — a crash between the two commands loses the in-flight job.",
+			"at-most-once — a crash between the two commands loses the in-flight job." +
+			(optedIn
+				? "\n  Running this way in PRODUCTION because allowNonAtomicPop was set."
+				: ""),
 	);
 }
 
@@ -101,7 +124,7 @@ export class RedisDriver implements QueueDriver {
 		if (this.#resolved) return this.#resolved;
 		if (typeof this.#source !== "function") {
 			this.#resolved = this.#source;
-			warnWithoutLmove(this.#resolved);
+			checkLmove(this.#resolved, this.#allowNonAtomicPop);
 			return this.#resolved;
 		}
 		if (!this.#pending) {
@@ -109,7 +132,7 @@ export class RedisDriver implements QueueDriver {
 			this.#pending = Promise.resolve(resolver())
 				.then((client) => {
 					this.#resolved = client;
-					warnWithoutLmove(client);
+					checkLmove(client, this.#allowNonAtomicPop);
 					return client;
 				})
 				// Cleared on failure too. Clearing only on success left the
@@ -125,18 +148,30 @@ export class RedisDriver implements QueueDriver {
 
 	constructor(
 		source: RedisClientSource,
-		options?: { prefix?: string; visibilityTimeoutMs?: number },
+		options?: {
+			prefix?: string;
+			visibilityTimeoutMs?: number;
+			/**
+			 * Accept the non-atomic pop on a Redis older than 6.2, in
+			 * production. Off by default: losing an accepted job is a choice a
+			 * deployment makes, not one a version check makes for it.
+			 */
+			allowNonAtomicPop?: boolean;
+		},
 	) {
 		this.#source = source;
 		// A client handed in directly can be checked now, so the warning keeps
 		// landing at construction as it always did. A named connection has no
 		// client yet — it is checked when the connection resolves.
-		if (typeof source !== "function") warnWithoutLmove(source);
+		if (typeof source !== "function") {
+			checkLmove(source, options?.allowNonAtomicPop ?? false);
+		}
 		// Normalised rather than documented: every key is built by concatenation
 		// (`${prefix}pending`), so a prefix without a trailing separator yields
 		// "myapppending" — unreadable, and able to collide with a neighbouring
 		// prefix. Nothing warned, because nothing failed.
 		this.#prefix = withSeparator(options?.prefix ?? "queue:");
+		this.#allowNonAtomicPop = options?.allowNonAtomicPop ?? false;
 		const visibilityTimeout = options?.visibilityTimeoutMs ?? 30_000;
 		// A non-positive / non-integer timeout makes pop()'s `SET … PX <ms>` fail
 		// on a real Redis; the catch then removes the job from `processing` and
@@ -152,6 +187,7 @@ export class RedisDriver implements QueueDriver {
 
 	#pendingKey = () => `${this.#prefix}pending`;
 	#processingKey = () => `${this.#prefix}processing`;
+	#allowNonAtomicPop = false;
 	#failedKey = () => `${this.#prefix}failed`;
 	#leaseKey = (jobId: string) => `${this.#prefix}lease:${jobId}`;
 
@@ -177,26 +213,37 @@ export class RedisDriver implements QueueDriver {
 		}
 
 		if (!raw) return null;
+
+		// Only a payload that can never be run is purged. Everything past this
+		// point is a REAL job that already sits in `processing`, and deleting
+		// it there is the one thing that loses it for good: it is gone from
+		// pending too, and recoverStale() scans processing, so nothing would
+		// ever find it again.
+		let parsed: unknown;
 		try {
-			const parsed: unknown = JSON.parse(raw);
-			if (!isValidJob(parsed)) {
-				// Malformed payload — purge from `processing` so it can't sit
-				// there indefinitely as a poison pill. recoverStale() also
-				// catches survivors but pop()'s own move is the primary path.
-				await client.lrem(this.#processingKey(), 1, raw);
-				return null;
-			}
-			await client.set(
-				this.#leaseKey(parsed.id),
-				raw,
-				"PX",
-				String(this.#visibilityTimeout),
-			);
-			return parsed;
+			parsed = JSON.parse(raw);
 		} catch {
+			// A poison pill: unparseable, and it would sit in processing
+			// forever blocking nothing but wasting every recovery pass.
+			await client.lrem(this.#processingKey(), 1, raw);
+			return null;
+		}
+		if (!isValidJob(parsed)) {
 			await client.lrem(this.#processingKey(), 1, raw);
 			return null;
 		}
+
+		// A lease that cannot be written is a transient Redis failure, not a
+		// bad job. The error propagates and the job STAYS in processing with
+		// no lease, which is precisely the state recoverStale() puts back in
+		// pending — so the delivery guarantee survives the blip.
+		await client.set(
+			this.#leaseKey(parsed.id),
+			raw,
+			"PX",
+			String(this.#visibilityTimeout),
+		);
+		return parsed;
 	}
 
 	async complete(job: Job): Promise<void> {

package/src/index.ts

@@ -12,6 +12,7 @@ export type { RedisClient } from "./drivers/RedisDriver.js";
 export { RedisDriver } from "./drivers/RedisDriver.js";
 export type { Job, JobHandler, QueueDriver } from "./QueueManager.js";
 export { QueueManager } from "./QueueManager.js";
+export { type QueueStoreFactory, stores } from "./stores.js";
 
 import type { BayProviderConfig } from "./BayProvider.js";
 

package/src/nodeEnv.ts

@@ -0,0 +1,30 @@
+/**
+ * Reading `NODE_ENV`, with the aliases people actually set.
+ *
+ * `NODE_ENV=prod` is ordinary in a Dockerfile or a platform dashboard. Read
+ * verbatim it answers "not production" — and here that decides whether a queue
+ * silently accepts a delivery guarantee weaker than the one it advertises.
+ *
+ * Duplicated rather than imported: bay depends on no other package in this
+ * workspace, and a safety decision that only holds when an optional peer is
+ * installed is not a decision.
+ */
+
+const DEV_ENVS = ["dev", "develop", "development"];
+const PROD_ENVS = ["prod", "production"];
+const TEST_ENVS = ["test", "testing"];
+
+/** The canonical name for whatever `NODE_ENV` holds. */
+export function normalizeNodeEnv(value: string | undefined): string {
+	if (!value || typeof value !== "string") return "unknown";
+	const env = value.toLowerCase();
+	if (DEV_ENVS.includes(env)) return "development";
+	if (PROD_ENVS.includes(env)) return "production";
+	if (TEST_ENVS.includes(env)) return "test";
+	return env;
+}
+
+/** Whether this process is running in production, under any spelling. */
+export function inProduction(): boolean {
+	return normalizeNodeEnv(process.env.NODE_ENV) === "production";
+}

package/src/services/main.ts

@@ -27,6 +27,14 @@ export function getQueue(): QueueManager | undefined {
 	return instance;
 }
 
+/**
+ * @internal Release the singleton, so a shut-down application does not leave a
+ * dead queue reachable through `services/main`.
+ */
+export function clearQueue(): void {
+	instance = undefined;
+}
+
 const queue: QueueManager = new Proxy({} as QueueManager, {
 	get(_target, prop) {
 		// A module loader inspects what it imports before anyone uses it: it reads

package/src/stores.ts

@@ -0,0 +1,59 @@
+/**
+ * The queue store factories a config file names — `{ default, stores }`.
+ *
+ * The shape AdonisJS gives a package with several backends and one selected:
+ * a `stores` namespace imported beside `defineConfig`, each entry that
+ * namespace's result, and the selection read from the environment. Bay had a
+ * single `driver: "memory"` and told everyone else to build the manager by
+ * hand, which meant Redis could not be reached from a config file at all.
+ *
+ *   import { defineConfig, stores } from '@c9up/bay'
+ *
+ *   export default defineConfig({
+ *     default: env.get('QUEUE_STORE'),
+ *     stores: {
+ *       memory: stores.memory(),
+ *       redis:  stores.redis({ connection: 'main' }),
+ *     },
+ *   })
+ *
+ * Factories are lazy: only the store an application actually uses is built, so
+ * naming a Redis queue in a config that runs in memory opens no connection.
+ */
+
+import { MemoryDriver } from "./drivers/MemoryDriver.js";
+import type { RedisClientSource } from "./drivers/RedisDriver.js";
+import { RedisDriver } from "./drivers/RedisDriver.js";
+import type { QueueDriver } from "./QueueManager.js";
+import { quasarConnection } from "./quasar.js";
+
+/** A driver, built on first use. */
+export type QueueStoreFactory = () => QueueDriver;
+
+export const stores = {
+	/** In memory. Jobs do not survive a restart — for tests and dev. */
+	memory(): QueueStoreFactory {
+		return () => new MemoryDriver();
+	},
+
+	/**
+	 * Redis. `connection` takes an ioredis-shaped client, a function answering
+	 * one, or the NAME of a `@c9up/quasar` connection — the last resolved at
+	 * first use, without bay importing quasar, which stays an optional peer.
+	 */
+	redis(options: {
+		connection: RedisClientSource | string;
+		prefix?: string;
+		visibilityTimeoutMs?: number;
+	}): QueueStoreFactory {
+		const source: RedisClientSource =
+			typeof options.connection === "string"
+				? quasarConnection(options.connection)
+				: options.connection;
+		return () =>
+			new RedisDriver(source, {
+				prefix: options.prefix,
+				visibilityTimeoutMs: options.visibilityTimeoutMs,
+			});
+	},
+};