@warlock.js/queue 5.13.0

package/esm/queue-manager.mjs

@@ -0,0 +1,109 @@
+import { getQueueConfig } from "./config.mjs";
+import { onJobRegistered, queueOf, registeredJobs } from "./job-registry.mjs";
+import { processJob } from "./process-job.mjs";
+import { log } from "@warlock.js/logger";
+import { Queue, Worker } from "bullmq";
+
+//#region ../queue/src/queue-manager.ts
+const DEFAULT_SHUTDOWN_TIMEOUT = 3e4;
+const queues = /* @__PURE__ */ new Map();
+const workers = /* @__PURE__ */ new Map();
+let stopListening;
+/**
+* The BullMQ queue for `name`, created on first use with the configured
+* connection and prefix.
+*/
+function getQueue(name) {
+	let queue = queues.get(name);
+	if (!queue) {
+		const config = getQueueConfig();
+		queue = new Queue(name, {
+			connection: config.connection,
+			prefix: config.prefix ?? "warlock"
+		});
+		queue.on("error", (error) => {
+			log.error("queue", "connection", error);
+		});
+		queues.set(name, queue);
+	}
+	return queue;
+}
+/**
+* Start one worker per queue that has a registered job, and keep starting
+* workers for queues whose first job is defined later.
+*
+* A no-op returning `[]` when `workers.enabled` is `false`. Calling it again
+* while workers run starts only the missing ones.
+*
+* @returns the queue names that now have a worker in this process.
+*/
+async function startWorkers() {
+	if (getQueueConfig().workers?.enabled === false) return [];
+	for (const job of registeredJobs()) ensureWorker(queueOf(job));
+	stopListening ??= onJobRegistered((job) => {
+		ensureWorker(queueOf(job));
+	});
+	await Promise.all([...workers.values()].map((worker) => worker.waitUntilReady()));
+	return [...workers.keys()];
+}
+/** The queue names with a running worker in this process. */
+function runningWorkers() {
+	return [...workers.keys()];
+}
+function ensureWorker(queueName) {
+	if (workers.has(queueName)) return;
+	const config = getQueueConfig();
+	const worker = new Worker(queueName, processJob, {
+		connection: config.connection,
+		prefix: config.prefix ?? "warlock",
+		concurrency: config.workers?.concurrency ?? 1
+	});
+	worker.on("error", (error) => {
+		log.error("queue", "worker", error);
+	});
+	worker.on("failed", (job, error) => {
+		log.error("queue", "job.failed", `${job?.name ?? "unknown"} (${job?.id ?? "?"}): ${error.message}`);
+	});
+	workers.set(queueName, worker);
+}
+/**
+* Graceful shutdown: stop workers taking new jobs and wait for active ones
+* (bounded by `timeout`, then force-close), then close every queue
+* connection. Safe to call when nothing was started, and more than once.
+*/
+async function closeQueue(options = {}) {
+	stopListening?.();
+	stopListening = void 0;
+	const timeout = options.timeout ?? configuredShutdownTimeout();
+	const closingWorkers = [...workers.values()];
+	workers.clear();
+	await Promise.all(closingWorkers.map((worker) => closeWorker(worker, timeout)));
+	const closingQueues = [...queues.values()];
+	queues.clear();
+	await Promise.all(closingQueues.map((queue) => queue.close()));
+}
+function configuredShutdownTimeout() {
+	try {
+		return getQueueConfig().workers?.shutdownTimeout ?? DEFAULT_SHUTDOWN_TIMEOUT;
+	} catch {
+		return DEFAULT_SHUTDOWN_TIMEOUT;
+	}
+}
+async function closeWorker(worker, timeout) {
+	let timer;
+	const timedOut = new Promise((resolve) => {
+		timer = setTimeout(() => resolve("timeout"), timeout);
+	});
+	const closing = worker.close().then(() => "closed");
+	closing.catch(() => void 0);
+	const outcome = await Promise.race([closing, timedOut]);
+	clearTimeout(timer);
+	if (outcome === "timeout") {
+		log.warn("queue", "shutdown", `Worker for "${worker.name}" still had active jobs after ${timeout}ms; disconnecting. Those jobs are retried once their lock expires.`);
+		await worker.disconnect();
+	}
+}
+
+//#endregion
+export { closeQueue, getQueue, runningWorkers, startWorkers };
+//# sourceMappingURL=queue-manager.mjs.map

package/esm/queue-manager.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue-manager.mjs","names":[],"sources":["../../../../../../queue/src/queue-manager.ts"],"sourcesContent":["import { log } from \"@warlock.js/logger\";\nimport { Queue, Worker } from \"bullmq\";\nimport { getQueueConfig } from \"./config\";\nimport { onJobRegistered, queueOf, registeredJobs } from \"./job-registry\";\nimport { processJob } from \"./process-job\";\n\nconst DEFAULT_SHUTDOWN_TIMEOUT = 30_000;\n\nconst queues = new Map<string, Queue>();\nconst workers = new Map<string, Worker>();\nlet stopListening: (() => void) | undefined;\n\n/**\n * The BullMQ queue for `name`, created on first use with the configured\n * connection and prefix.\n */\nexport function getQueue(name: string): Queue {\n  let queue = queues.get(name);\n\n  if (!queue) {\n    const config = getQueueConfig();\n\n    queue = new Queue(name, {\n      connection: config.connection,\n      prefix: config.prefix ?? \"warlock\",\n    });\n\n    queue.on(\"error\", (error) => {\n      log.error(\"queue\", \"connection\", error);\n    });\n\n    queues.set(name, queue);\n  }\n\n  return queue;\n}\n\n/**\n * Start one worker per queue that has a registered job, and keep starting\n * workers for queues whose first job is defined later.\n *\n * A no-op returning `[]` when `workers.enabled` is `false`. Calling it again\n * while workers run starts only the missing ones.\n *\n * @returns the queue names that now have a worker in this process.\n */\nexport async function startWorkers(): Promise<string[]> {\n  const config = getQueueConfig();\n\n  if (config.workers?.enabled === false) {\n    return [];\n  }\n\n  for (const job of registeredJobs()) {\n    ensureWorker(queueOf(job));\n  }\n\n  stopListening ??= onJobRegistered((job) => {\n    ensureWorker(queueOf(job));\n  });\n\n  await Promise.all([...workers.values()].map((worker) => worker.waitUntilReady()));\n\n  return [...workers.keys()];\n}\n\n/** The queue names with a running worker in this process. */\nexport function runningWorkers(): string[] {\n  return [...workers.keys()];\n}\n\nfunction ensureWorker(queueName: string): void {\n  if (workers.has(queueName)) {\n    return;\n  }\n\n  const config = getQueueConfig();\n\n  const worker = new Worker(queueName, processJob, {\n    connection: config.connection,\n    prefix: config.prefix ?? \"warlock\",\n    concurrency: config.workers?.concurrency ?? 1,\n  });\n\n  worker.on(\"error\", (error) => {\n    log.error(\"queue\", \"worker\", error);\n  });\n\n  worker.on(\"failed\", (job, error) => {\n    log.error(\"queue\", \"job.failed\", `${job?.name ?? \"unknown\"} (${job?.id ?? \"?\"}): ${error.message}`);\n  });\n\n  workers.set(queueName, worker);\n}\n\nexport type CloseQueueOptions = {\n  /**\n   * How long to wait for active jobs before force-closing workers, in\n   * milliseconds. Default: `workers.shutdownTimeout`, else `30000`.\n   */\n  timeout?: number;\n};\n\n/**\n * Graceful shutdown: stop workers taking new jobs and wait for active ones\n * (bounded by `timeout`, then force-close), then close every queue\n * connection. Safe to call when nothing was started, and more than once.\n */\nexport async function closeQueue(options: CloseQueueOptions = {}): Promise<void> {\n  stopListening?.();\n  stopListening = undefined;\n\n  const timeout = options.timeout ?? configuredShutdownTimeout();\n  const closingWorkers = [...workers.values()];\n  workers.clear();\n\n  await Promise.all(closingWorkers.map((worker) => closeWorker(worker, timeout)));\n\n  const closingQueues = [...queues.values()];\n  queues.clear();\n\n  await Promise.all(closingQueues.map((queue) => queue.close()));\n}\n\nfunction configuredShutdownTimeout(): number {\n  try {\n    return getQueueConfig().workers?.shutdownTimeout ?? DEFAULT_SHUTDOWN_TIMEOUT;\n  } catch {\n    return DEFAULT_SHUTDOWN_TIMEOUT;\n  }\n}\n\nasync function closeWorker(worker: Worker, timeout: number): Promise<void> {\n  let timer: NodeJS.Timeout | undefined;\n\n  const timedOut = new Promise<\"timeout\">((resolve) => {\n    timer = setTimeout(() => resolve(\"timeout\"), timeout);\n  });\n\n  // A close already in progress cannot be upgraded to a forced one (BullMQ\n  // returns the pending promise), so the timeout path drops the connections\n  // instead and lets the graceful close settle in the background.\n  const closing = worker.close().then(() => \"closed\" as const);\n  closing.catch(() => undefined);\n\n  const outcome = await Promise.race([closing, timedOut]);\n\n  clearTimeout(timer);\n\n  if (outcome === \"timeout\") {\n    log.warn(\n      \"queue\",\n      \"shutdown\",\n      `Worker for \"${worker.name}\" still had active jobs after ${timeout}ms; disconnecting. ` +\n        \"Those jobs are retried once their lock expires.\",\n    );\n\n    await worker.disconnect();\n  }\n}\n"],"mappings":";;;;;;;AAMA,MAAM,2BAA2B;AAEjC,MAAM,yBAAS,IAAI,IAAmB;AACtC,MAAM,0BAAU,IAAI,IAAoB;AACxC,IAAI;;;;;AAMJ,SAAgB,SAAS,MAAqB;CAC5C,IAAI,QAAQ,OAAO,IAAI,IAAI;CAE3B,IAAI,CAAC,OAAO;EACV,MAAM,SAAS,eAAe;EAE9B,QAAQ,IAAI,MAAM,MAAM;GACtB,YAAY,OAAO;GACnB,QAAQ,OAAO,UAAU;EAC3B,CAAC;EAED,MAAM,GAAG,UAAU,UAAU;GAC3B,IAAI,MAAM,SAAS,cAAc,KAAK;EACxC,CAAC;EAED,OAAO,IAAI,MAAM,KAAK;CACxB;CAEA,OAAO;AACT;;;;;;;;;;AAWA,eAAsB,eAAkC;CAGtD,IAFe,eAEN,CAAC,CAAC,SAAS,YAAY,OAC9B,OAAO,CAAC;CAGV,KAAK,MAAM,OAAO,eAAe,GAC/B,aAAa,QAAQ,GAAG,CAAC;CAG3B,kBAAkB,iBAAiB,QAAQ;EACzC,aAAa,QAAQ,GAAG,CAAC;CAC3B,CAAC;CAED,MAAM,QAAQ,IAAI,CAAC,GAAG,QAAQ,OAAO,CAAC,CAAC,CAAC,KAAK,WAAW,OAAO,eAAe,CAAC,CAAC;CAEhF,OAAO,CAAC,GAAG,QAAQ,KAAK,CAAC;AAC3B;;AAGA,SAAgB,iBAA2B;CACzC,OAAO,CAAC,GAAG,QAAQ,KAAK,CAAC;AAC3B;AAEA,SAAS,aAAa,WAAyB;CAC7C,IAAI,QAAQ,IAAI,SAAS,GACvB;CAGF,MAAM,SAAS,eAAe;CAE9B,MAAM,SAAS,IAAI,OAAO,WAAW,YAAY;EAC/C,YAAY,OAAO;EACnB,QAAQ,OAAO,UAAU;EACzB,aAAa,OAAO,SAAS,eAAe;CAC9C,CAAC;CAED,OAAO,GAAG,UAAU,UAAU;EAC5B,IAAI,MAAM,SAAS,UAAU,KAAK;CACpC,CAAC;CAED,OAAO,GAAG,WAAW,KAAK,UAAU;EAClC,IAAI,MAAM,SAAS,cAAc,GAAG,KAAK,QAAQ,UAAU,IAAI,KAAK,MAAM,IAAI,KAAK,MAAM,SAAS;CACpG,CAAC;CAED,QAAQ,IAAI,WAAW,MAAM;AAC/B;;;;;;AAeA,eAAsB,WAAW,UAA6B,CAAC,GAAkB;CAC/E,gBAAgB;CAChB,gBAAgB;CAEhB,MAAM,UAAU,QAAQ,WAAW,0BAA0B;CAC7D,MAAM,iBAAiB,CAAC,GAAG,QAAQ,OAAO,CAAC;CAC3C,QAAQ,MAAM;CAEd,MAAM,QAAQ,IAAI,eAAe,KAAK,WAAW,YAAY,QAAQ,OAAO,CAAC,CAAC;CAE9E,MAAM,gBAAgB,CAAC,GAAG,OAAO,OAAO,CAAC;CACzC,OAAO,MAAM;CAEb,MAAM,QAAQ,IAAI,cAAc,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAC/D;AAEA,SAAS,4BAAoC;CAC3C,IAAI;EACF,OAAO,eAAe,CAAC,CAAC,SAAS,mBAAmB;CACtD,QAAQ;EACN,OAAO;CACT;AACF;AAEA,eAAe,YAAY,QAAgB,SAAgC;CACzE,IAAI;CAEJ,MAAM,WAAW,IAAI,SAAoB,YAAY;EACnD,QAAQ,iBAAiB,QAAQ,SAAS,GAAG,OAAO;CACtD,CAAC;CAKD,MAAM,UAAU,OAAO,MAAM,CAAC,CAAC,WAAW,QAAiB;CAC3D,QAAQ,YAAY,MAAS;CAE7B,MAAM,UAAU,MAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,CAAC;CAEtD,aAAa,KAAK;CAElB,IAAI,YAAY,WAAW;EACzB,IAAI,KACF,SACA,YACA,eAAe,OAAO,KAAK,gCAAgC,QAAQ,mEAErE;EAEA,MAAM,OAAO,WAAW;CAC1B;AACF"}

package/esm/types.d.mts

@@ -0,0 +1,149 @@
+import { ConnectionOptions } from "bullmq";
+
+//#region ../queue/src/types.d.ts
+/** Units accepted in a {@link Duration} string. */
+type DurationUnit = "ms" | "s" | "m" | "h" | "d";
+/**
+ * A duration: milliseconds as a number, or a string such as `"500ms"`,
+ * `"30s"`, `"10m"`, `"2h"`, `"1d"`.
+ */
+type Duration = number | `${number}${DurationUnit}`;
+/**
+ * How a failed attempt waits before the next one. A number is a fixed delay in
+ * milliseconds.
+ */
+type JobBackoff = number | {
+  /** `fixed` waits `delay` every time; `exponential` waits `delay * 2^(attempt - 1)`. */type: "fixed" | "exponential"; /** Base delay in milliseconds. */
+  delay: number;
+};
+/**
+ * How many finished jobs to keep: `true` removes them immediately, `false`
+ * keeps them all, a number keeps the newest N.
+ */
+type JobRetention = boolean | number;
+/**
+ * Options that shape how a job is retried and retained. Set app-wide in
+ * `queue.defaultJobOptions`, per job in `defineJob`.
+ */
+type JobOptions = {
+  /** Total attempts including the first. Default `1` (no retry). */attempts?: number; /** Wait between attempts. */
+  backoff?: JobBackoff; /** Retention for completed jobs. Default keeps them. */
+  removeOnComplete?: JobRetention; /** Retention for failed jobs. Default keeps them, so `failedJobs()` can list them. */
+  removeOnFail?: JobRetention;
+};
+/**
+ * In-process worker settings.
+ */
+type QueueWorkersConfig = {
+  /**
+   * Start workers in this process. Default `true`. Set `false` for a process
+   * that only dispatches jobs while another process consumes them.
+   */
+  enabled?: boolean; /** Jobs processed in parallel per queue. Default `1`. */
+  concurrency?: number;
+  /**
+   * How long shutdown waits for active jobs before force-closing the workers,
+   * in milliseconds. Default `30000`.
+   */
+  shutdownTimeout?: number;
+};
+/**
+ * The `queue` configuration key — `src/config/queue.ts`.
+ */
+type QueueConfig = {
+  /**
+   * Redis connection. Any BullMQ connection option: `{ host, port, password, db }`,
+   * `{ url }`, or an ioredis instance.
+   */
+  connection: ConnectionOptions; /** Redis key prefix for every queue. Default `"warlock"`. */
+  prefix?: string; /** Queue name used when a job does not name one. Default `"default"`. */
+  defaultQueue?: string; /** Defaults merged under every job's own options. */
+  defaultJobOptions?: JobOptions; /** In-process workers. */
+  workers?: QueueWorkersConfig;
+};
+/** A progress value: a number (e.g. a percentage) or a JSON object. */
+type JobProgress = number | Record<string, unknown>;
+/**
+ * What a job handler receives beside its payload.
+ */
+type JobContext = {
+  /** The job id. */id: string; /** The job name given to `defineJob`. */
+  name: string; /** The queue the job runs on. */
+  queue: string; /** The current attempt, starting at `1`. */
+  attempt: number; /** Total attempts allowed. */
+  maxAttempts: number; /** Record progress; readable through `job.find(id)`. */
+  progress(value: JobProgress): Promise<void>; /** Append a line to the job's log. */
+  log(line: string): Promise<void>;
+};
+/**
+ * The definition passed to `defineJob`.
+ */
+type JobDefinition<TPayload, TResult> = JobOptions & {
+  /** Unique job name, e.g. `"invoices.send"`. */name: string; /** Queue to run on. Default: `queue.defaultQueue` (`"default"`). */
+  queue?: string; /** The work. Throw to fail the attempt; the return value is stored as the job result. */
+  handle(payload: TPayload, context: JobContext): Promise<TResult> | TResult;
+};
+/**
+ * Per-dispatch options.
+ */
+type DispatchOptions = {
+  /** Wait before the job becomes available. */delay?: Duration;
+  /**
+   * Priority: `1` is the highest; larger numbers run later. Omit for no
+   * priority — such jobs run ahead of every prioritized job.
+   */
+  priority?: number;
+  /**
+   * Explicit job id. Dispatching again with an id that still exists is a
+   * no-op, which makes dispatch idempotent.
+   */
+  jobId?: string; /** Override the job's attempts for this dispatch. */
+  attempts?: number; /** Override the job's backoff for this dispatch. */
+  backoff?: JobBackoff;
+};
+/** What `dispatch` resolves with. */
+type DispatchedJob = {
+  id: string;
+  name: string;
+  queue: string;
+};
+/** A job's lifecycle state. */
+type JobState = "waiting" | "delayed" | "prioritized" | "active" | "completed" | "failed" | "waiting-children" | "unknown";
+/** A point-in-time view of a job. */
+type JobSnapshot<TPayload = unknown, TResult = unknown> = {
+  id: string;
+  name: string;
+  queue: string;
+  state: JobState;
+  payload: TPayload;
+  progress: JobProgress;
+  attemptsMade: number;
+  result?: TResult;
+  failedReason?: string;
+  createdAt: Date;
+  finishedAt?: Date;
+};
+/** A failed job, with the means to retry it. */
+type FailedJob<TPayload = unknown> = {
+  id: string;
+  name: string;
+  queue: string;
+  payload: TPayload;
+  attemptsMade: number;
+  failedReason: string;
+  stacktrace: string[];
+  failedAt?: Date; /** Move the job back to waiting so a worker picks it up again. */
+  retry(): Promise<void>;
+};
+/**
+ * A job returned by `defineJob`.
+ */
+type QueueJob<TPayload, TResult = unknown> = {
+  readonly name: string;
+  readonly queue: string; /** Enqueue the job. */
+  dispatch(payload: TPayload, options?: DispatchOptions): Promise<DispatchedJob>; /** Read a job of this type by id; `undefined` when it no longer exists. */
+  find(id: string): Promise<JobSnapshot<TPayload, TResult> | undefined>;
+};
+//#endregion
+export { DispatchOptions, DispatchedJob, Duration, DurationUnit, FailedJob, JobBackoff, JobContext, JobDefinition, JobOptions, JobProgress, JobRetention, JobSnapshot, JobState, QueueConfig, QueueJob, QueueWorkersConfig };
+//# sourceMappingURL=types.d.mts.map

package/esm/types.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.mts","names":[],"sources":["../../../../../../queue/src/types.ts"],"mappings":";;;;KAGY,YAAA;AAAZ;;;;AAAA,KAMY,QAAA,wBAAgC,YAAY;AAAxD;;;;AAAA,KAMY,UAAA;EAAA,uFAIN,IAAA;EAEA,KAAK;AAAA;AAOX;;;;AAAA,KAAY,YAAA;AAMZ;;;;AAAA,KAAY,UAAA;EAQK,kEANf,QAAA,WAM2B;EAJ3B,OAAA,GAAU,UAAA,EAAV;EAEA,gBAAA,GAAmB,YAAA,EAAnB;EAEA,YAAA,GAAe,YAAA;AAAA;;;AAAY;KAMjB,kBAAA;EAAkB;;;;EAK5B,OAAA,YAOA;EALA,WAAA;EAKe;AAMjB;;;EANE,eAAA;AAAA;;;;KAMU,WAAA;EAKE;;;;EAAZ,UAAA,EAAY,iBAAA,EAQZ;EANA,MAAA,WAM4B;EAJ5B,YAAA,WAQU;EANV,iBAAA,GAAoB,UAAA;EAEpB,OAAA,GAAU,kBAAA;AAAA;AASZ;AAAA,KALY,WAAA,YAAuB,MAAM;;;;KAK7B,UAAA;EAcgB,kBAZ1B,EAAA;EAEA,IAAA,UAEA;EAAA,KAAA,UAIA;EAFA,OAAA,UAIgB;EAFhB,WAAA,UAE8B;EAA9B,QAAA,CAAS,KAAA,EAAO,WAAA,GAAc,OAAA,QAE1B;EAAJ,GAAA,CAAI,IAAA,WAAe,OAAA;AAAA;AAAO;AAM5B;;AAN4B,KAMhB,aAAA,sBAAmC,UAAA;EAAA,+CAE7C,IAAA,UAImC;EAFnC,KAAA,WAEgD;EAAhD,MAAA,CAAO,OAAA,EAAS,QAAA,EAAU,OAAA,EAAS,UAAA,GAAa,OAAA,CAAQ,OAAA,IAAW,OAAA;AAAA;;;;KAMzD,eAAA;EAVV,6CAYA,KAAA,GAAQ,QAAA;EARR;;;;EAaA,QAAA;EAbgD;;;;EAkBhD,KAAA,WAZU;EAcV,QAAA;EAEA,OAAA,GAAU,UAAU;AAAA;;KAIV,aAAA;EACV,EAAA;EACA,IAAA;EACA,KAAA;AAAA;;KAIU,QAAA;AAPZ;AAAA,KAkBY,WAAA;EACV,EAAA;EACA,IAAA;EACA,KAAA;EACA,KAAA,EAAO,QAAA;EACP,OAAA,EAAS,QAAA;EACT,QAAA,EAAU,WAAA;EACV,YAAA;EACA,MAAA,GAAS,OAAA;EACT,YAAA;EACA,SAAA,EAAW,IAAA;EACX,UAAA,GAAa,IAAA;AAAA;AAXf;AAAA,KAeY,SAAA;EACV,EAAA;EACA,IAAA;EACA,KAAA;EACA,OAAA,EAAS,QAAA;EACT,YAAA;EACA,YAAA;EACA,UAAA;EACA,QAAA,GAAW,IAAA,EAZM;EAcjB,KAAA,IAAS,OAAA;AAAA;;;;KAMC,QAAA;EAAA,SACD,IAAA;EAAA,SACA,KAAA,UA5BA;EA8BT,QAAA,CAAS,OAAA,EAAS,QAAA,EAAU,OAAA,GAAU,eAAA,GAAkB,OAAA,CAAQ,aAAA,GA7BtD;EA+BV,IAAA,CAAK,EAAA,WAAa,OAAA,CAAQ,WAAA,CAAY,QAAA,EAAU,OAAA;AAAA"}

package/llms-full.txt

@@ -0,0 +1,229 @@
+# Warlock Queue — full skills
+
+> Package: `@warlock.js/queue`
+
+> Generated artifact. Concatenates every SKILL.md and reference file under `@warlock.js/queue/skills/`. Re-run `node scripts/generate-llms.mjs` after any change.
+
+## configure-queue  `@warlock.js/queue/configure-queue/SKILL.md`
+
+---
+name: configure-queue
+description: 'Configure `@warlock.js/queue`: the declarative `src/config/queue.ts` (`QueueConfig` — `connection`, `prefix`, `defaultQueue`, `defaultJobOptions`, `workers: { enabled, concurrency, shutdownTimeout }`), registering `queueConnector()` in `warlock.config.ts > connectors`, running a dispatch-only process, and graceful shutdown. Programmatic `setQueueConfig` / `startWorkers` / `closeQueue` for scripts and tests. Triggers: `QueueConfig`, `queueConnector`, `setQueueConfig`, `startWorkers`, `closeQueue`, `workers.enabled`, `shutdownTimeout`; "configure the queue", "connect BullMQ to Redis", "disable workers in the web process", "graceful shutdown of jobs". Skip: writing jobs — `@warlock.js/queue/define-jobs/SKILL.md`.'
+---
+
+# Configure the queue
+
+Redis is required. The config is declarative; the connector applies it.
+
+```ts title="src/config/queue.ts"
+import type { QueueConfig } from "@warlock.js/queue";
+
+const queueConfig: QueueConfig = {
+  connection: { host: "127.0.0.1", port: 6379 }, // any BullMQ connection option, or { url }
+  prefix: "my-app", // Redis key prefix, default "warlock"
+  defaultJobOptions: { attempts: 3, backoff: { type: "exponential", delay: 1000 } },
+  workers: {
+    enabled: true, // default true
+    concurrency: 5, // jobs in parallel per queue, default 1
+    shutdownTimeout: 30_000, // ms to wait for active jobs, default 30000
+  },
+};
+
+export default queueConfig;
+```
+
+```ts title="warlock.config.ts"
+import { defineConfig } from "@warlock.js/core";
+import { queueConnector } from "@warlock.js/queue";
+
+export default defineConfig({
+  connectors: [queueConnector()],
+});
+```
+
+## What the connector does
+
+- Starts in the **late** phase — after app code is imported — so every `defineJob` has registered. One worker per queue that has a job.
+- No `queue` config → logs a warning and does nothing.
+- On SIGINT/SIGTERM: workers stop taking jobs, active jobs get up to `shutdownTimeout` ms, then Redis connections close. A job still running after that is retried by another worker once its lock expires.
+
+## Dispatch-only process
+
+Set `workers.enabled: false`. `dispatch()` still works; nothing is processed in that process. Run the workers elsewhere with the same job definitions:
+
+```ts
+import { setQueueConfig, startWorkers, closeQueue } from "@warlock.js/queue";
+import "./jobs"; // modules that call defineJob
+
+setQueueConfig({ connection: { host: "127.0.0.1", port: 6379 } });
+await startWorkers();
+process.on("SIGTERM", () => closeQueue());
+```
+
+Every process must use the same `prefix`, or they will not see each other's jobs.
+
+
+## define-jobs  `@warlock.js/queue/define-jobs/SKILL.md`
+
+---
+name: define-jobs
+description: 'Define and dispatch background jobs with `@warlock.js/queue`: `defineJob({ name, queue?, attempts?, backoff?, removeOnComplete?, removeOnFail?, handle(payload, ctx) })` returns a typed job; `job.dispatch(payload, { delay, priority, jobId, attempts, backoff })`; the handler context (`id`, `attempt`, `maxAttempts`, `progress()`, `log()`); reading a job with `job.find(id)`. Triggers: `defineJob`, `.dispatch(`, `ctx.progress`, `JobContext`, `DispatchOptions`, `backoff`, `attempts`, `priority`, `jobId`; "run this in the background", "retry with backoff", "delay a job", "job progress", "idempotent dispatch". Skip: config and workers — `@warlock.js/queue/configure-queue/SKILL.md`; failed jobs — `@warlock.js/queue/manage-failed-jobs/SKILL.md`.'
+---
+
+# Define and dispatch jobs
+
+```ts title="src/app/invoices/jobs/send-invoice.job.ts"
+import { defineJob } from "@warlock.js/queue";
+
+export const sendInvoice = defineJob({
+  name: "invoices.send", // unique across the app
+  attempts: 5, // total tries, default 1
+  backoff: { type: "exponential", delay: 2000 }, // or a number = fixed ms
+  async handle(payload: { invoiceId: string }, ctx) {
+    await ctx.log(`attempt ${ctx.attempt} of ${ctx.maxAttempts}`);
+    await ctx.progress(50);
+    // throw to fail this attempt; it is retried until attempts run out
+    return { sent: true }; // stored as the job result
+  },
+});
+```
+
+The job module must be imported by the process that runs workers — in a Warlock app, anything under `src/app` that is loaded at boot.
+
+## Dispatch
+
+```ts
+const { id } = await sendInvoice.dispatch({ invoiceId: "42" });
+
+await sendInvoice.dispatch({ invoiceId: "43" }, {
+  delay: "10m", // ms number or "500ms" | "30s" | "10m" | "2h" | "1d"
+  priority: 1, // 1 runs first; larger numbers later
+  jobId: "invoice:43", // a second dispatch with a live id is ignored
+});
+```
+
+Option precedence: dispatch options > `defineJob` > `queue.defaultJobOptions`.
+
+## Read a job
+
+```ts
+const job = await sendInvoice.find(id);
+// { id, name, queue, state, payload, progress, attemptsMade, result, failedReason, createdAt, finishedAt }
+```
+
+## Rules
+
+- A job name with no handler in the worker process fails at once, without retries.
+- Payloads are stored as JSON: pass ids, not model instances.
+- Defining the same name again replaces the handler (this is what keeps dev reloads working), so keep names unique.
+
+
+## manage-failed-jobs  `@warlock.js/queue/manage-failed-jobs/SKILL.md`
+
+---
+name: manage-failed-jobs
+description: 'Inspect and retry failed `@warlock.js/queue` jobs: `failedJobs({ queue, start, end })` returns `FailedJob[]` (`id`, `name`, `payload`, `attemptsMade`, `failedReason`, `stacktrace`, `failedAt`, `retry()`), `retryFailedJob(id, { queue })` (throws `FailedJobNotFoundError`), and the optional bull-board UI via `queueDashboard(server, { basePath, queues })` — needs `@bull-board/api` + `@bull-board/fastify`, loaded only on call, missing ones throw `QueueDashboardDependencyError`. Triggers: `failedJobs`, `retryFailedJob`, `queueDashboard`, `bull-board`; "list failed jobs", "retry a failed job", "queue dashboard", "job admin UI". Skip: defining retries — `@warlock.js/queue/define-jobs/SKILL.md`.'
+---
+
+# Failed jobs
+
+A job is failed once it has used every attempt, or failed in a way that cannot be retried (for example, no handler for its name). Failed jobs are kept unless you set `removeOnFail`.
+
+```ts
+import { failedJobs, retryFailedJob } from "@warlock.js/queue";
+
+const failed = await failedJobs({ queue: "default", start: 0, end: 49 }); // newest first
+
+for (const job of failed) {
+  console.log(job.name, job.failedReason, job.attemptsMade);
+}
+
+await failed[0]?.retry(); // back to waiting
+await retryFailedJob("invoice:43"); // by id; throws FailedJobNotFoundError if not failed
+```
+
+## Dashboard (optional)
+
+```sh
+npm install @bull-board/api @bull-board/fastify
+```
+
+```ts
+import { getHttpServer } from "@warlock.js/core";
+import { queueDashboard } from "@warlock.js/queue";
+
+await queueDashboard(getHttpServer(), { basePath: "/admin/queues" });
+```
+
+- Call it before the HTTP server starts listening.
+- Shows every queue that has a job, plus the default queue, unless you pass `queues`.
+- It has no authentication of its own, and it can retry and delete jobs. Protect the path.
+
+
+## overview  `@warlock.js/queue/overview/SKILL.md`
+
+---
+name: overview
+description: 'Front door for `@warlock.js/queue` — durable background jobs for Warlock apps on BullMQ + Redis (Redis is required): `defineJob` + `.dispatch()`, retries/backoff, delay, priority, progress, failed-job listing/retry, in-process workers started by `queueConnector()` with graceful shutdown, a BullMQ backend for notifications `.queue()`, and an optional bull-board dashboard. TRIGGER when: importing from `@warlock.js/queue`; "background job", "job queue", "run this later", "retry failed jobs", "BullMQ in Warlock", "worker process". Skip: in-memory batching inside one process — that is core''s `Queue` class (`@warlock.js/core`); cron-style schedules — `@warlock.js/scheduler/overview/SKILL.md`; a known task — load `configure-queue`, `define-jobs`, `manage-failed-jobs` or `queue-notifications`.'
+---
+
+# `@warlock.js/queue` — overview
+
+Durable background jobs. Jobs are stored in **Redis** (required), retried on failure, and processed by workers — inside the app process by default.
+
+## Mental model
+
+- **Job definition** — `defineJob({ name, handle })` registers a handler by name and returns a typed job.
+- **Dispatch** — `job.dispatch(payload, options)` stores the job in Redis. Any process with the same definition and a worker can run it.
+- **Worker** — one per queue name, started by `queueConnector()` (or `startWorkers()`); routes each job to the handler with its name.
+- **Shutdown** — workers stop taking jobs, active jobs finish (up to `workers.shutdownTimeout`), then connections close.
+
+## Not core's `Queue`
+
+`@warlock.js/core` exports `Queue` — an in-memory batcher that flushes items by size or interval inside one process. No storage, no retries, lost on exit. Use `@warlock.js/queue` when work must survive a restart or be retried.
+
+## Skills index
+
+- [`configure-queue`](@warlock.js/queue/configure-queue/SKILL.md) — `src/config/queue.ts`, `queueConnector()`, workers on/off, shutdown.
+- [`define-jobs`](@warlock.js/queue/define-jobs/SKILL.md) — `defineJob`, `dispatch` options, retries, progress, `find`.
+- [`manage-failed-jobs`](@warlock.js/queue/manage-failed-jobs/SKILL.md) — `failedJobs`, `retryFailedJob`, the bull-board dashboard.
+- [`queue-notifications`](@warlock.js/queue/queue-notifications/SKILL.md) — `queueNotificationDispatcher()` for notifications `.queue()`.
+
+
+## queue-notifications  `@warlock.js/queue/queue-notifications/SKILL.md`
+
+---
+name: queue-notifications
+description: 'Send `@warlock.js/notifications` `.queue()` deliveries through BullMQ with `queueNotificationDispatcher({ queue?, attempts?, backoff? })` from `@warlock.js/queue/notifications` — put it in the `queue` slot of `src/config/notifications.ts`; the queue workers deliver. Honours `SendOptions.delay` (number = seconds, or "10m"), retries a failing `channel.send`, and fails at once for a channel missing from the worker config. Notifications keeps no queue dependency. Triggers: `queueNotificationDispatcher`, `@warlock.js/queue/notifications`, `NotificationConfig.queue`; "queue notifications with BullMQ", "retry notification delivery", "delayed notification". Skip: the herald backend — `@warlock.js/notifications/queue-notifications/SKILL.md`.'
+---
+
+# Queue notifications with BullMQ
+
+```ts title="src/config/notifications.ts"
+import { type NotificationConfig, mailChannel } from "@warlock.js/notifications";
+import { queueNotificationDispatcher } from "@warlock.js/queue/notifications";
+
+const config: NotificationConfig = {
+  channels: { mail: mailChannel() },
+  queue: queueNotificationDispatcher({ attempts: 3, backoff: { type: "exponential", delay: 5000 } }),
+};
+
+export default config;
+```
+
+Also configure the queue itself (`configure-queue`). No separate notifications worker is needed: the delivery is an ordinary job, run by the queue workers.
+
+```ts
+await orderShipped.queue(user, { order }); // enqueued, delivered by a worker
+await orderShipped.queue(user, { order }, { delay: "10m" }); // delivered in 10 minutes
+```
+
+## Behaviour
+
+- The job carries the rendered payload and resolved route; the worker looks the channel up by name in its own notifications config and calls `channel.send`.
+- `delay`: a number is seconds (notifications' convention); a string is a duration.
+- `channel.send` throws → retried per `attempts` / `backoff`, then listed by `failedJobs()`.
+- Channel not configured in the worker → fails at once, no retries.
+- Deliveries run under the job name `warlock.notifications.deliver`.
+
+

package/llms.txt

@@ -0,0 +1,13 @@
+# Warlock Queue
+
+> Package: `@warlock.js/queue`
+
+> Durable background jobs for Warlock.js on BullMQ + Redis: defineJob, dispatch with delay/priority, retries with backoff, progress, failed-job access, in-process workers with graceful shutdown.
+
+## Skills
+
+- [configure-queue](@warlock.js/queue/configure-queue/SKILL.md): Configure `@warlock.js/queue`: the declarative `src/config/queue.ts` (`QueueConfig` — `connection`, `prefix`, `defaultQueue`, `defaultJobOptions`, `workers: { enabled, concurrency, shutdownTimeout }`), registering `queueConnector()` in `warlock.config.ts > connectors`, running a dispatch-only process, and graceful shutdown. Programmatic `setQueueConfig` / `startWorkers` / `closeQueue` for scripts and tests. Triggers: `QueueConfig`, `queueConnector`, `setQueueConfig`, `startWorkers`, `closeQueue`, `workers.enabled`, `shutdownTimeout`; "configure the queue", "connect BullMQ to Redis", "disable workers in the web process", "graceful shutdown of jobs". Skip: writing jobs — `@warlock.js/queue/define-jobs/SKILL.md`.
+- [define-jobs](@warlock.js/queue/define-jobs/SKILL.md): Define and dispatch background jobs with `@warlock.js/queue`: `defineJob({ name, queue?, attempts?, backoff?, removeOnComplete?, removeOnFail?, handle(payload, ctx) })` returns a typed job; `job.dispatch(payload, { delay, priority, jobId, attempts, backoff })`; the handler context (`id`, `attempt`, `maxAttempts`, `progress()`, `log()`); reading a job with `job.find(id)`. Triggers: `defineJob`, `.dispatch(`, `ctx.progress`, `JobContext`, `DispatchOptions`, `backoff`, `attempts`, `priority`, `jobId`; "run this in the background", "retry with backoff", "delay a job", "job progress", "idempotent dispatch". Skip: config and workers — `@warlock.js/queue/configure-queue/SKILL.md`; failed jobs — `@warlock.js/queue/manage-failed-jobs/SKILL.md`.
+- [manage-failed-jobs](@warlock.js/queue/manage-failed-jobs/SKILL.md): Inspect and retry failed `@warlock.js/queue` jobs: `failedJobs({ queue, start, end })` returns `FailedJob[]` (`id`, `name`, `payload`, `attemptsMade`, `failedReason`, `stacktrace`, `failedAt`, `retry()`), `retryFailedJob(id, { queue })` (throws `FailedJobNotFoundError`), and the optional bull-board UI via `queueDashboard(server, { basePath, queues })` — needs `@bull-board/api` + `@bull-board/fastify`, loaded only on call, missing ones throw `QueueDashboardDependencyError`. Triggers: `failedJobs`, `retryFailedJob`, `queueDashboard`, `bull-board`; "list failed jobs", "retry a failed job", "queue dashboard", "job admin UI". Skip: defining retries — `@warlock.js/queue/define-jobs/SKILL.md`.
+- [overview](@warlock.js/queue/overview/SKILL.md): Front door for `@warlock.js/queue` — durable background jobs for Warlock apps on BullMQ + Redis (Redis is required): `defineJob` + `.dispatch()`, retries/backoff, delay, priority, progress, failed-job listing/retry, in-process workers started by `queueConnector()` with graceful shutdown, a BullMQ backend for notifications `.queue()`, and an optional bull-board dashboard. TRIGGER when: importing from `@warlock.js/queue`; "background job", "job queue", "run this later", "retry failed jobs", "BullMQ in Warlock", "worker process". Skip: in-memory batching inside one process — that is core's `Queue` class (`@warlock.js/core`); cron-style schedules — `@warlock.js/scheduler/overview/SKILL.md`; a known task — load `configure-queue`, `define-jobs`, `manage-failed-jobs` or `queue-notifications`.
+- [queue-notifications](@warlock.js/queue/queue-notifications/SKILL.md): Send `@warlock.js/notifications` `.queue()` deliveries through BullMQ with `queueNotificationDispatcher({ queue?, attempts?, backoff? })` from `@warlock.js/queue/notifications` — put it in the `queue` slot of `src/config/notifications.ts`; the queue workers deliver. Honours `SendOptions.delay` (number = seconds, or "10m"), retries a failing `channel.send`, and fails at once for a channel missing from the worker config. Notifications keeps no queue dependency. Triggers: `queueNotificationDispatcher`, `@warlock.js/queue/notifications`, `NotificationConfig.queue`; "queue notifications with BullMQ", "retry notification delivery", "delayed notification". Skip: the herald backend — `@warlock.js/notifications/queue-notifications/SKILL.md`.

package/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "@warlock.js/queue",
+  "description": "Durable background jobs for Warlock.js on BullMQ + Redis: defineJob, dispatch with delay/priority, retries with backoff, progress, failed-job access, in-process workers with graceful shutdown.",
+  "warlock": {
+    "environment": "server"
+  },
+  "dependencies": {
+    "bullmq": "^6.3.6",
+    "ioredis": "^5.11.1"
+  },
+  "peerDependencies": {
+    "@bull-board/api": "^9.0.0",
+    "@bull-board/fastify": "^9.0.0",
+    "@warlock.js/core": "5.13.0",
+    "@warlock.js/logger": "5.13.0",
+    "@warlock.js/notifications": "5.13.0"
+  },
+  "peerDependenciesMeta": {
+    "@bull-board/api": {
+      "optional": true
+    },
+    "@bull-board/fastify": {
+      "optional": true
+    },
+    "@warlock.js/notifications": {
+      "optional": true
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/warlockjs/queue"
+  },
+  "keywords": [
+    "warlock.js",
+    "queue",
+    "jobs",
+    "background",
+    "bullmq",
+    "redis",
+    "worker"
+  ],
+  "author": "hassanzohdy",
+  "license": "MIT",
+  "version": "5.13.0",
+  "main": "./cjs/index.cjs",
+  "module": "./esm/index.mjs",
+  "types": "./esm/index.d.mts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./esm/index.d.mts",
+        "default": "./esm/index.mjs"
+      },
+      "require": {
+        "types": "./esm/index.d.mts",
+        "default": "./cjs/index.cjs"
+      }
+    },
+    "./notifications": {
+      "import": {
+        "types": "./esm/notifications/index.d.mts",
+        "default": "./esm/notifications/index.mjs"
+      },
+      "require": {
+        "types": "./esm/notifications/index.d.mts",
+        "default": "./cjs/notifications/index.cjs"
+      }
+    }
+  }
+}

package/skills/configure-queue/SKILL.md

@@ -0,0 +1,55 @@
+---
+name: configure-queue
+description: 'Configure `@warlock.js/queue`: the declarative `src/config/queue.ts` (`QueueConfig` — `connection`, `prefix`, `defaultQueue`, `defaultJobOptions`, `workers: { enabled, concurrency, shutdownTimeout }`), registering `queueConnector()` in `warlock.config.ts > connectors`, running a dispatch-only process, and graceful shutdown. Programmatic `setQueueConfig` / `startWorkers` / `closeQueue` for scripts and tests. Triggers: `QueueConfig`, `queueConnector`, `setQueueConfig`, `startWorkers`, `closeQueue`, `workers.enabled`, `shutdownTimeout`; "configure the queue", "connect BullMQ to Redis", "disable workers in the web process", "graceful shutdown of jobs". Skip: writing jobs — `@warlock.js/queue/define-jobs/SKILL.md`.'
+---
+
+# Configure the queue
+
+Redis is required. The config is declarative; the connector applies it.
+
+```ts title="src/config/queue.ts"
+import type { QueueConfig } from "@warlock.js/queue";
+
+const queueConfig: QueueConfig = {
+  connection: { host: "127.0.0.1", port: 6379 }, // any BullMQ connection option, or { url }
+  prefix: "my-app", // Redis key prefix, default "warlock"
+  defaultJobOptions: { attempts: 3, backoff: { type: "exponential", delay: 1000 } },
+  workers: {
+    enabled: true, // default true
+    concurrency: 5, // jobs in parallel per queue, default 1
+    shutdownTimeout: 30_000, // ms to wait for active jobs, default 30000
+  },
+};
+
+export default queueConfig;
+```
+
+```ts title="warlock.config.ts"
+import { defineConfig } from "@warlock.js/core";
+import { queueConnector } from "@warlock.js/queue";
+
+export default defineConfig({
+  connectors: [queueConnector()],
+});
+```
+
+## What the connector does
+
+- Starts in the **late** phase — after app code is imported — so every `defineJob` has registered. One worker per queue that has a job.
+- No `queue` config → logs a warning and does nothing.
+- On SIGINT/SIGTERM: workers stop taking jobs, active jobs get up to `shutdownTimeout` ms, then Redis connections close. A job still running after that is retried by another worker once its lock expires.
+
+## Dispatch-only process
+
+Set `workers.enabled: false`. `dispatch()` still works; nothing is processed in that process. Run the workers elsewhere with the same job definitions:
+
+```ts
+import { setQueueConfig, startWorkers, closeQueue } from "@warlock.js/queue";
+import "./jobs"; // modules that call defineJob
+
+setQueueConfig({ connection: { host: "127.0.0.1", port: 6379 } });
+await startWorkers();
+process.on("SIGTERM", () => closeQueue());
+```
+
+Every process must use the same `prefix`, or they will not see each other's jobs.