@zudojs/queue 1.0.0 → 1.2.0

package/README.md

@@ -2,6 +2,12 @@
 
 Background job and asynchronous task infrastructure with in-memory and adapter-based queue implementations.
 
+<!-- zudo-docs:start -->
+
+**Documentation:** [zudojs.oyinlola.site/docs/packages-queue](https://zudojs.oyinlola.site/docs/packages-queue) · **For AI agents:** [Markdown version](https://zudojs.oyinlola.site/docs/packages-queue.md), [llms.txt](https://zudojs.oyinlola.site/llms.txt)
+
+<!-- zudo-docs:end -->
+
 ## Installation
 
 ```bash
@@ -43,9 +49,6 @@ await queue.add(
 // The queue polls in the background; give it a tick before reading counts.
 await new Promise((resolve) => setTimeout(resolve, 100));
 
-// The queue polls in the background; give it a tick before reading counts.
-await new Promise((resolve) => setTimeout(resolve, 100));
-
 // Jobs that exhaust their attempts land here rather than vanishing.
 console.log((await queue.getDeadLetterJobs()).length);
 console.log(await queue.getStats());
@@ -69,6 +72,58 @@ await queue.close();
 - Bounded retention of settled jobs, so a long-lived queue does not grow without
   limit
 
+## Workers, timeouts and context
+
+**One consumer at a time.** `queue.process()` registers a processor and, by
+default, the queue's own poller runs jobs. Creating a `Worker` for the queue
+turns that poller off as a consumer (`queue.setAutoProcess(false)`), so the
+worker's `middleware`, `timeoutMs` and `concurrency` apply to every job, and
+`worker.stop()` really stops consumption. Pass `autoProcess: false` to keep the
+queue from consuming before any worker exists.
+
+**Timeouts are cooperative.** A timeout aborts `context.signal`; honour it. The
+job's concurrency slot and its retry wait up to `timeoutGraceMs` (5000 ms by
+default) for the processor to settle, so a retry never runs beside the attempt
+it replaces. A processor still running after the grace period is abandoned.
+
+**Carrying context across the queue.** AsyncLocalStorage does not follow a job
+into the poller or worker that runs it. A `QueueContextCarrier` captures a value
+at `add()` into job metadata and restores it around the middleware and the
+processor:
+
+```typescript
+import { AsyncLocalStorage } from "node:async_hooks";
+import { createInMemoryQueue, createQueueName } from "@zudojs/queue";
+import type { QueueContextCarrier } from "@zudojs/queue";
+
+const tenants = new AsyncLocalStorage<{ tenantId: string }>();
+
+const tenantCarrier: QueueContextCarrier<string> = {
+  key: "tenantId",
+  capture: () => tenants.getStore()?.tenantId,
+  restore: (tenantId, run) => tenants.run({ tenantId }, run),
+};
+
+const jobs = createInMemoryQueue<{ id: number }>(createQueueName("reports"), {
+  contextCarriers: [tenantCarrier],
+});
+jobs.process("build", async () => {
+  console.log("building for", tenants.getStore()?.tenantId); // "acme"
+});
+
+await tenants.run({ tenantId: "acme" }, () => jobs.add("build", { id: 1 }));
+await new Promise((resolve) => setTimeout(resolve, 100));
+await jobs.close();
+```
+
+Keep captured values small and serializable (ids, not live objects). With
+`@zudojs/tenancy`, capture the tenant id and restore it with the tenant context
+storage's `run`.
+
+Errors with no caller to receive them (a worker's failing poll, a throwing event
+listener) go to `logger.error` when a logger is configured, and otherwise to
+`process.emitWarning`, never to `console`.
+
 ## Use Cases
 
 - Background email sending

package/dist/contextCarrier/contextCarrier.core.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Context capture at `add()` and restoration around processing.
+ *
+ * @module contextCarrier/contextCarrier.core
+ */
+import type { Job } from "../job/job.type.js";
+import type { QueueContextCarrier } from "./contextCarrier.type.js";
+/**
+ * Job metadata key under which captured context values are stored, as a
+ * `{ [carrier.key]: value }` record.
+ */
+export declare const CONTEXT_METADATA_KEY = "zudo:context";
+/**
+ * Captures every carrier's value into a metadata record.
+ *
+ * @param carriers - The queue's context carriers.
+ * @param metadata - The job's own metadata, if any.
+ * @returns The metadata with a {@link CONTEXT_METADATA_KEY} entry added, or
+ *   the input unchanged when no carrier captured anything.
+ */
+export declare function captureContext(carriers: readonly QueueContextCarrier[] | undefined, metadata: Record<string, unknown> | undefined): Record<string, unknown> | undefined;
+/**
+ * Runs `fn` inside every context the job carries, first carrier outermost.
+ *
+ * @param carriers - The queue's context carriers.
+ * @param job - The job about to run.
+ * @param fn - The work to run inside the restored context.
+ * @returns Whatever `fn` resolves to.
+ */
+export declare function runWithContext<T>(carriers: readonly QueueContextCarrier[] | undefined, job: Job<unknown>, fn: () => Promise<T>): Promise<T>;
+//# sourceMappingURL=contextCarrier.core.d.ts.map

package/dist/contextCarrier/contextCarrier.core.js

@@ -0,0 +1,60 @@
+/**
+ * Context capture at `add()` and restoration around processing.
+ *
+ * @module contextCarrier/contextCarrier.core
+ */
+/**
+ * Job metadata key under which captured context values are stored, as a
+ * `{ [carrier.key]: value }` record.
+ */
+export const CONTEXT_METADATA_KEY = "zudo:context";
+/**
+ * Captures every carrier's value into a metadata record.
+ *
+ * @param carriers - The queue's context carriers.
+ * @param metadata - The job's own metadata, if any.
+ * @returns The metadata with a {@link CONTEXT_METADATA_KEY} entry added, or
+ *   the input unchanged when no carrier captured anything.
+ */
+export function captureContext(carriers, metadata) {
+    if (!carriers || carriers.length === 0)
+        return metadata;
+    const captured = {};
+    let any = false;
+    for (const carrier of carriers) {
+        const value = carrier.capture();
+        if (value === undefined)
+            continue;
+        captured[carrier.key] = value;
+        any = true;
+    }
+    if (!any)
+        return metadata;
+    return { ...metadata, [CONTEXT_METADATA_KEY]: Object.freeze(captured) };
+}
+/**
+ * Runs `fn` inside every context the job carries, first carrier outermost.
+ *
+ * @param carriers - The queue's context carriers.
+ * @param job - The job about to run.
+ * @param fn - The work to run inside the restored context.
+ * @returns Whatever `fn` resolves to.
+ */
+export function runWithContext(carriers, job, fn) {
+    const raw = job.metadata?.[CONTEXT_METADATA_KEY];
+    if (!carriers || carriers.length === 0)
+        return fn();
+    if (typeof raw !== "object" || raw === null)
+        return fn();
+    const stored = raw;
+    let run = fn;
+    for (let i = carriers.length - 1; i >= 0; i--) {
+        const carrier = carriers[i];
+        if (!Object.hasOwn(stored, carrier.key))
+            continue;
+        const inner = run;
+        run = () => carrier.restore(stored[carrier.key], inner);
+    }
+    return run();
+}
+//# sourceMappingURL=contextCarrier.core.js.map

package/dist/contextCarrier/contextCarrier.type.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Context carrier contract.
+ *
+ * @module contextCarrier/contextCarrier.type
+ */
+/**
+ * Captures one piece of ambient context when a job is added and restores it
+ * while the job runs.
+ *
+ * AsyncLocalStorage does not follow a job from the request that enqueued it
+ * into the poller or worker that runs it, so without a carrier every job runs
+ * with no tenant, no correlation id and no trace. Keep the captured value
+ * small and serializable (an id, not a live object): a broker-backed queue
+ * has to store it with the job.
+ *
+ * @example
+ * ```typescript
+ * const tenantCarrier: QueueContextCarrier<string> = {
+ *   key: "tenantId",
+ *   capture: () => tenantStorage.get()?.tenant?.id,
+ *   restore: (tenantId, run) =>
+ *     tenantStorage.run(contextFor(tenantId), run),
+ * };
+ * createInMemoryQueue("emails", { contextCarriers: [tenantCarrier] });
+ * ```
+ */
+export interface QueueContextCarrier<TValue = unknown> {
+    /** Unique name under which the value is stored with the job. */
+    readonly key: string;
+    /**
+     * Reads the value from the caller's context at `add()`. Returning
+     * `undefined` stores nothing.
+     */
+    capture(): TValue | undefined;
+    /**
+     * Runs `run` inside the restored context. Called only for jobs that
+     * carry a value for this carrier's `key`.
+     */
+    restore<T>(value: TValue, run: () => Promise<T>): Promise<T>;
+}
+//# sourceMappingURL=contextCarrier.type.d.ts.map

package/dist/contextCarrier/contextCarrier.type.js

@@ -0,0 +1,7 @@
+/**
+ * Context carrier contract.
+ *
+ * @module contextCarrier/contextCarrier.type
+ */
+export {};
+//# sourceMappingURL=contextCarrier.type.js.map

package/dist/contextCarrier/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * @zudojs/queue/contextCarrier
+ *
+ * Carries ambient execution context (tenant, correlation id, trace ids)
+ * across the queue boundary: captured into job metadata at `add()` and
+ * restored around the processor, so a background job runs in the context of
+ * the request that enqueued it.
+ */
+export type { QueueContextCarrier } from "./contextCarrier.type.js";
+export { CONTEXT_METADATA_KEY, captureContext, runWithContext, } from "./contextCarrier.core.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/contextCarrier/index.js

@@ -0,0 +1,10 @@
+/**
+ * @zudojs/queue/contextCarrier
+ *
+ * Carries ambient execution context (tenant, correlation id, trace ids)
+ * across the queue boundary: captured into job metadata at `add()` and
+ * restored around the processor, so a background job runs in the context of
+ * the request that enqueued it.
+ */
+export { CONTEXT_METADATA_KEY, captureContext, runWithContext, } from "./contextCarrier.core.js";
+//# sourceMappingURL=index.js.map

package/dist/inMemoryQueue/inMemoryQueue.core.d.ts

@@ -20,6 +20,8 @@ export declare class InMemoryQueue<TData = unknown> implements Queue<TData> {
     private readonly middleware;
     private paused;
     private disposed;
+    /** Whether the internal poller claims jobs; see `setAutoProcess`. */
+    private autoProcess;
     private activeCount;
     private pollTimer;
     private readonly scheduledTimers;
@@ -35,6 +37,15 @@ export declare class InMemoryQueue<TData = unknown> implements Queue<TData> {
     private emptySince;
     private backoffMs;
     constructor(name: QueueName, options?: QueueOptions);
+    /**
+     * Turns the internal poller on or off as a consumer.
+     *
+     * While off, the poller still promotes scheduled jobs and reclaims stalled
+     * ones, but claims nothing: an external `Worker` is the only consumer, so
+     * stopping it really stops consumption and its middleware, timeout and
+     * concurrency apply to every job.
+     */
+    setAutoProcess(enabled: boolean): void;
     add(jobName: string, data: TData, options?: JobOptions): Promise<Job<TData>>;
     process(name: string, processor: Processor<TData>): void;
     getJob(jobId: JobId): Promise<Job<TData> | null>;

package/dist/inMemoryQueue/inMemoryQueue.core.js

@@ -7,6 +7,7 @@ import { JsonSerializer } from "../serializer/serializer.core.js";
 import { createInMemoryDeadLetterStore } from "../deadLetter/deadLetter.core.js";
 import { createNoopQueueEventEmitter } from "../queueEmitter/queueEmitter.core.js";
 import { processJob } from "./inMemoryQueue.processing.js";
+import { captureContext } from "../contextCarrier/contextCarrier.core.js";
 import { scheduleJob, promoteDueScheduledJobs, } from "./inMemoryQueue.scheduling.js";
 /** Terminal states a job never leaves. */
 const TERMINAL_STATES = new Set([
@@ -34,6 +35,8 @@ export class InMemoryQueue {
     middleware;
     paused = false;
     disposed = false;
+    /** Whether the internal poller claims jobs; see `setAutoProcess`. */
+    autoProcess;
     activeCount = 0;
     pollTimer = null;
     scheduledTimers = new Map();
@@ -62,6 +65,18 @@ export class InMemoryQueue {
         this.emitter = options?.eventEmitter ?? createNoopQueueEventEmitter();
         this.deadLetterStore =
             this.options.deadLetterStore ?? createInMemoryDeadLetterStore();
+        this.autoProcess = this.options.autoProcess ?? true;
+    }
+    /**
+     * Turns the internal poller on or off as a consumer.
+     *
+     * While off, the poller still promotes scheduled jobs and reclaims stalled
+     * ones, but claims nothing: an external `Worker` is the only consumer, so
+     * stopping it really stops consumption and its middleware, timeout and
+     * concurrency apply to every job.
+     */
+    setAutoProcess(enabled) {
+        this.autoProcess = enabled;
     }
     async add(jobName, data, options) {
         if (this.disposed)
@@ -74,7 +89,9 @@ export class InMemoryQueue {
                 queueName: this.name,
             });
         }
-        const mergedOptions = { ...this.options.defaultJobOptions, ...options };
+        const merged = { ...this.options.defaultJobOptions, ...options };
+        const metadata = captureContext(this.options.contextCarriers, merged.metadata);
+        const mergedOptions = metadata === merged.metadata ? merged : { ...merged, metadata };
         if (mergedOptions.deduplicationKey) {
             const existing = this.deduplicationIndex.get(mergedOptions.deduplicationKey);
             if (existing && this.jobs.has(existing)) {
@@ -402,6 +419,12 @@ export class InMemoryQueue {
             await processJob(job, processor, {
                 timeoutMs: job.timeoutMs ?? options?.timeoutMs,
                 abortController,
+                ...(this.options.timeoutGraceMs !== undefined
+                    ? { timeoutGraceMs: this.options.timeoutGraceMs }
+                    : {}),
+                ...(this.options.contextCarriers
+                    ? { contextCarriers: this.options.contextCarriers }
+                    : {}),
             }, {
                 jobs: this.jobs,
                 emitter: this.emitter,
@@ -413,6 +436,9 @@ export class InMemoryQueue {
                 registerRetryTimer: (jobId, timer) => {
                     this.retryTimers.set(jobId, timer);
                 },
+                deregisterRetryTimer: (jobId) => {
+                    this.retryTimers.delete(jobId);
+                },
                 onSettled: (settled) => this.recordSettled(settled),
                 isDisposed: () => this.disposed,
                 ...(this.options.logger ? { logger: this.options.logger } : {}),
@@ -427,7 +453,6 @@ export class InMemoryQueue {
         finally {
             this.activeCount--;
             this.inFlight.delete(job.id);
-            this.retryTimers.delete(job.id);
         }
     }
     async processTick() {
@@ -435,7 +460,7 @@ export class InMemoryQueue {
             return;
         const concurrency = Math.max(1, this.options.concurrency ?? 1);
         let processed = 0;
-        while (this.activeCount < concurrency) {
+        while (this.autoProcess && this.activeCount < concurrency) {
             // `claimNextJob` only returns jobs that have a registered
             // processor and moves them out of `waiting`, so this loop always
             // terminates. Returning an unrunnable job here is what previously

package/dist/inMemoryQueue/inMemoryQueue.processing.d.ts

@@ -5,6 +5,7 @@ import type { QueueMiddleware } from "../middleware/middleware.type.js";
 import type { QueueEventEmitter } from "../queueEmitter/queueEmitter.type.js";
 import type { DeadLetterStore } from "../deadLetter/deadLetter.type.js";
 import type { QueueLogger } from "../queue/queue.type.js";
+import type { QueueContextCarrier } from "../contextCarrier/contextCarrier.type.js";
 /**
  * Mutable throughput counters.
  *
@@ -32,6 +33,8 @@ export interface ProcessJobDependencies<TData> {
      * queue can clear it on close instead of leaking it.
      */
     readonly registerRetryTimer: (jobId: JobId, timer: ReturnType<typeof setTimeout>) => void;
+    /** Forgets a retry timer once it has fired. */
+    readonly deregisterRetryTimer?: (jobId: JobId) => void;
     /** Invoked whenever a job reaches a terminal state. */
     readonly onSettled?: (job: Job<TData>) => void;
     /** Whether the owning queue has been disposed. */
@@ -52,6 +55,10 @@ export interface ProcessJobDependencies<TData> {
 export declare function processJob<TData>(job: Job<TData>, processor: Processor<TData>, options: {
     timeoutMs?: number;
     abortController?: AbortController;
+    /** See `QueueOptions.timeoutGraceMs`. */
+    timeoutGraceMs?: number;
+    /** See `QueueOptions.contextCarriers`. */
+    contextCarriers?: readonly QueueContextCarrier[];
 }, deps: ProcessJobDependencies<TData>): Promise<void>;
 /**
  * Handle job failure with retry logic.

package/dist/inMemoryQueue/inMemoryQueue.processing.js

@@ -5,6 +5,8 @@ import { createJobContext } from "../jobContext/jobContext.core.js";
 import { createMiddlewareChain, createTimeoutMiddleware, } from "../middleware/middleware.core.js";
 import { calculateRetryDelay, resolveBackoff, shouldRetry, } from "../retryPolicy/retryPolicy.core.js";
 import { moveToDeadLetter } from "../deadLetter/deadLetter.core.js";
+import { runWithContext } from "../contextCarrier/contextCarrier.core.js";
+import { DEFAULT_TIMEOUT_GRACE_MS, settleWithin, } from "./inMemoryQueue.settle.js";
 import { JobMaxAttemptsError } from "@zudojs/errors";
 /**
  * Narrows a processor's return value to a `JobResult`.
@@ -57,14 +59,18 @@ export async function processJob(job, processor, options, deps) {
         timeoutMiddleware,
         ...deps.middleware,
     ]);
+    // The processor promise itself, so a failure (a timeout above all) can
+    // wait for it to stop before the slot and the retry are released.
+    let running;
     try {
-        const result = await middlewareChain({
+        const result = await runWithContext(options.contextCarriers, updatedJob, () => middlewareChain({
             job: updatedJob,
             context,
             next: async () => {
-                return processor(updatedJob, context);
+                running = Promise.resolve(processor(updatedJob, context));
+                return running;
             },
-        });
+        }));
         if (isJobResult(result) && !result.success) {
             await handleJobFailure(updatedJob, result.error ?? "Job failed", deps);
         }
@@ -85,6 +91,7 @@ export async function processJob(job, processor, options, deps) {
     }
     catch (error) {
         const errorMessage = error instanceof Error ? error.message : String(error);
+        await settleWithin(running, options.timeoutGraceMs ?? DEFAULT_TIMEOUT_GRACE_MS);
         await handleJobFailure(updatedJob, errorMessage, deps);
     }
     finally {
@@ -122,6 +129,7 @@ export async function handleJobFailure(job, errorMessage, deps) {
         const backoff = resolveBackoff(incrementedJob.backoff);
         const delay = calculateRetryDelay(incrementedJob.attempt, backoff);
         const timer = setTimeout(() => {
+            deps.deregisterRetryTimer?.(job.id);
             if (deps.isDisposed()) {
                 return;
             }

package/dist/inMemoryQueue/inMemoryQueue.settle.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Waiting for a timed-out processor to actually stop.
+ *
+ * The timeout middleware rejects as soon as the timer fires, but the
+ * processor promise it raced keeps running until it notices the aborted
+ * signal. Freeing the concurrency slot and scheduling the retry at that
+ * moment let the retry run beside the original attempt.
+ *
+ * @module inMemoryQueue/inMemoryQueue.settle
+ */
+/** Default grace for a timed-out processor to settle, in milliseconds. */
+export declare const DEFAULT_TIMEOUT_GRACE_MS = 5000;
+/**
+ * Resolves once `pending` settles or `graceMs` elapses, whichever is first.
+ *
+ * Never rejects: the processor's own outcome has already been superseded by
+ * the failure being handled.
+ *
+ * @param pending - The processor promise, if the processor was started.
+ * @param graceMs - The longest to wait.
+ * @returns True when the processor settled within the grace period.
+ */
+export declare function settleWithin(pending: Promise<unknown> | undefined, graceMs: number): Promise<boolean>;
+//# sourceMappingURL=inMemoryQueue.settle.d.ts.map

package/dist/inMemoryQueue/inMemoryQueue.settle.js

@@ -0,0 +1,39 @@
+/**
+ * Waiting for a timed-out processor to actually stop.
+ *
+ * The timeout middleware rejects as soon as the timer fires, but the
+ * processor promise it raced keeps running until it notices the aborted
+ * signal. Freeing the concurrency slot and scheduling the retry at that
+ * moment let the retry run beside the original attempt.
+ *
+ * @module inMemoryQueue/inMemoryQueue.settle
+ */
+/** Default grace for a timed-out processor to settle, in milliseconds. */
+export const DEFAULT_TIMEOUT_GRACE_MS = 5_000;
+/**
+ * Resolves once `pending` settles or `graceMs` elapses, whichever is first.
+ *
+ * Never rejects: the processor's own outcome has already been superseded by
+ * the failure being handled.
+ *
+ * @param pending - The processor promise, if the processor was started.
+ * @param graceMs - The longest to wait.
+ * @returns True when the processor settled within the grace period.
+ */
+export async function settleWithin(pending, graceMs) {
+    if (pending === undefined)
+        return true;
+    let timer;
+    const settled = pending.then(() => true, () => true);
+    const expired = new Promise((resolve) => {
+        timer = setTimeout(() => resolve(false), Math.max(0, graceMs));
+    });
+    try {
+        return await Promise.race([settled, expired]);
+    }
+    finally {
+        if (timer !== undefined)
+            clearTimeout(timer);
+    }
+}
+//# sourceMappingURL=inMemoryQueue.settle.js.map

package/dist/index.d.ts

@@ -23,4 +23,5 @@ export * from "./middleware/index.js";
 export * from "./worker/index.js";
 export * from "./inMemoryQueue/index.js";
 export * from "./deadLetter/index.js";
+export * from "./contextCarrier/index.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -39,4 +39,6 @@ export * from "./worker/index.js";
 export * from "./inMemoryQueue/index.js";
 // Dead letter
 export * from "./deadLetter/index.js";
+// Context propagation across the queue boundary
+export * from "./contextCarrier/index.js";
 //# sourceMappingURL=index.js.map

package/dist/queue/queue.report.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Last-resort reporting for failures nobody else can receive.
+ *
+ * A worker's poll failure or a throwing event listener has no caller to
+ * reject. They used to go to `console.error`, bypassing structured logging
+ * and redaction. They now go to the configured logger's `error`, and without
+ * one to `process.emitWarning`, which Node prints once and applications can
+ * intercept with `process.on("warning")`.
+ *
+ * @module queue/queue.report
+ */
+import type { QueueLogger } from "./queue.type.js";
+/**
+ * Report an error that has no caller to receive it.
+ *
+ * @param message - What failed.
+ * @param error - The failure.
+ * @param logger - The configured logger, if any.
+ */
+export declare function reportQueueError(message: string, error: unknown, logger?: QueueLogger): void;
+//# sourceMappingURL=queue.report.d.ts.map

package/dist/queue/queue.report.js

@@ -0,0 +1,26 @@
+/**
+ * Last-resort reporting for failures nobody else can receive.
+ *
+ * A worker's poll failure or a throwing event listener has no caller to
+ * reject. They used to go to `console.error`, bypassing structured logging
+ * and redaction. They now go to the configured logger's `error`, and without
+ * one to `process.emitWarning`, which Node prints once and applications can
+ * intercept with `process.on("warning")`.
+ *
+ * @module queue/queue.report
+ */
+/**
+ * Report an error that has no caller to receive it.
+ *
+ * @param message - What failed.
+ * @param error - The failure.
+ * @param logger - The configured logger, if any.
+ */
+export function reportQueueError(message, error, logger) {
+    if (logger?.error) {
+        logger.error(message, { error });
+        return;
+    }
+    process.emitWarning(error instanceof Error ? error : new Error(String(error)), { type: "ZudoQueueWarning", detail: message });
+}
+//# sourceMappingURL=queue.report.js.map

package/dist/queue/queue.type.d.ts

@@ -5,6 +5,7 @@ import type { Processor } from "../processor/processor.type.js";
 import type { Serializer } from "../serializer/serializer.type.js";
 import type { QueueMiddleware } from "../middleware/middleware.type.js";
 import type { QueueEventEmitter } from "../queueEmitter/queueEmitter.type.js";
+import type { QueueContextCarrier } from "../contextCarrier/contextCarrier.type.js";
 import type { DeadLetterJob, DeadLetterStore } from "../deadLetter/deadLetter.type.js";
 /**
  * Somewhere for a job to write a log line.
@@ -13,6 +14,8 @@ import type { DeadLetterJob, DeadLetterStore } from "../deadLetter/deadLetter.ty
  */
 export interface QueueLogger {
     info(message: string, data?: Record<string, unknown>): void;
+    /** Receives failures that have no caller to reject (poll errors). */
+    error?(message: string, data?: Record<string, unknown>): void;
 }
 /**
  * Options for creating a queue.
@@ -79,6 +82,28 @@ export interface QueueOptions {
      * reclaimed again. Defaults to 3.
      */
     readonly maxStalledCount?: number;
+    /**
+     * Whether the queue's own poller claims and runs jobs. Defaults to
+     * `true`. Creating a `Worker` for the queue turns it off, so the worker —
+     * with its middleware, timeout and concurrency — is the only consumer.
+     */
+    readonly autoProcess?: boolean;
+    /**
+     * After a job times out, how long its concurrency slot and its retry wait
+     * for the processor to actually settle, in milliseconds. Defaults to 5000.
+     *
+     * A timeout aborts `context.signal`; a processor that ignores the signal
+     * keeps running. Without the wait, the retry started beside it and one
+     * job ran several times at once. Past this grace the processor is
+     * abandoned and the job fails anyway, so processors must honour the
+     * signal.
+     */
+    readonly timeoutGraceMs?: number;
+    /**
+     * Context carried from `add()` into the processor (tenant, correlation
+     * id, trace ids). See {@link QueueContextCarrier}.
+     */
+    readonly contextCarriers?: readonly QueueContextCarrier[];
 }
 /**
  * Statistics for a queue.
@@ -171,6 +196,11 @@ export interface Queue<TData = unknown> {
     getDeadLetterJobs(): Promise<readonly DeadLetterJob<TData>[]>;
     /** Close the queue, draining in-flight jobs first. */
     close(): Promise<void>;
+    /**
+     * Turns the queue's own poller on or off as a consumer. `createWorker`
+     * turns it off so a worker is never racing the queue for jobs.
+     */
+    setAutoProcess?(enabled: boolean): void;
 }
 /**
  * Event types emitted by a queue.

package/dist/queueEmitter/queueEmitter.core.js

@@ -1,3 +1,4 @@
+import { reportQueueError } from "../queue/queue.report.js";
 /**
  * In-memory queue event emitter.
  *
@@ -14,7 +15,7 @@ export class InMemoryQueueEventEmitter {
             options.onHandlerError ??
                 ((error, event) => {
                     queueMicrotask(() => {
-                        console.error(`[@zudojs/queue] Listener for "${event}" threw.`, error);
+                        reportQueueError(`[@zudojs/queue] Listener for "${event}" threw.`, error);
                     });
                 });
     }

package/dist/worker/worker.core.d.ts

@@ -6,6 +6,11 @@ import type { Worker, WorkerOptions } from "./worker.type.js";
  * The worker claims each job before running it, so a job is never picked
  * up twice — by this worker on its next poll, or by another worker on the
  * same queue.
+ *
+ * Creating a worker turns the queue's own poller off as a consumer
+ * (`queue.setAutoProcess(false)`), so the worker — with its middleware,
+ * timeout and concurrency — is the only thing running jobs, and `stop()`
+ * really stops consumption.
  */
 export declare function createWorker<TData>(id: string, queue: Queue<TData>, options?: WorkerOptions): Worker<TData>;
 /**

package/dist/worker/worker.core.js

@@ -1,5 +1,6 @@
 import { JobState as JobStateEnum, WorkerState, } from "../jobTypes/jobTypes.type.js";
 import { WorkerLifecycleError } from "@zudojs/errors";
+import { reportQueueError } from "../queue/queue.report.js";
 /** How long `stop()` waits for in-flight jobs before forcing a stop. */
 const DEFAULT_DRAIN_TIMEOUT_MS = 30_000;
 /**
@@ -8,6 +9,11 @@ const DEFAULT_DRAIN_TIMEOUT_MS = 30_000;
  * The worker claims each job before running it, so a job is never picked
  * up twice — by this worker on its next poll, or by another worker on the
  * same queue.
+ *
+ * Creating a worker turns the queue's own poller off as a consumer
+ * (`queue.setAutoProcess(false)`), so the worker — with its middleware,
+ * timeout and concurrency — is the only thing running jobs, and `stop()`
+ * really stops consumption.
  */
 export function createWorker(id, queue, options) {
     let state = WorkerState.CREATED;
@@ -17,17 +23,29 @@ export function createWorker(id, queue, options) {
     const drainTimeout = options?.drainTimeout ?? DEFAULT_DRAIN_TIMEOUT_MS;
     let pollTimer = null;
     let activeJobs = 0;
+    let polling = false;
     let abortController = null;
+    queue.setAutoProcess?.(false);
     const onError = options?.onError ??
         ((error) => {
             queueMicrotask(() => {
-                console.error(`[@zudojs/queue] Worker "${id}" poll failed.`, error);
+                reportQueueError(`[@zudojs/queue] Worker "${id}" poll failed.`, error, options?.logger);
             });
         });
+    /**
+     * Arms the next poll. At most one timer is ever armed: a delayed poll
+     * already pending is left alone, while an immediate poll (capacity just
+     * freed up, or a job was just dispatched) supersedes it.
+     */
     const scheduleNextPoll = (delay) => {
         if (state !== WorkerState.RUNNING) {
             return;
         }
+        if (pollTimer !== null) {
+            if (delay > 0)
+                return;
+            clearTimeout(pollTimer);
+        }
         pollTimer = setTimeout(runPoll, delay);
         pollTimer.unref?.();
     };
@@ -37,34 +55,20 @@ export function createWorker(id, queue, options) {
      * fatal, would take down the whole application.
      */
     const runPoll = () => {
+        pollTimer = null;
         void poll().catch((error) => {
             onError(error);
             scheduleNextPoll(pollInterval);
         });
     };
-    const poll = async () => {
-        if (state !== WorkerState.RUNNING || abortController?.signal.aborted) {
-            return;
-        }
-        if (activeJobs >= concurrency) {
-            scheduleNextPoll(pollInterval);
-            return;
-        }
-        const job = await queue.claimNextJob();
-        if (!job) {
-            scheduleNextPoll(pollInterval);
-            return;
-        }
-        const proc = queue.getProcessor(job.name);
-        if (!proc) {
-            // Claimed but unrunnable: release it rather than stranding it in
-            // `active` where nothing would ever pick it up again.
-            await queue.releaseJob(job.id);
-            scheduleNextPoll(pollInterval);
-            return;
-        }
-        activeJobs++;
-        stats.processed++;
+    /**
+     * Runs one claimed job to completion and frees its concurrency slot.
+     *
+     * Deliberately not awaited by `poll`: awaiting it there serialised the
+     * worker, so `concurrency` was reported by `getStats()` and honoured by
+     * nothing.
+     */
+    const runClaimedJob = async (job) => {
         try {
             // Dispatch through the queue rather than invoking the processor
             // directly. The queue owns job state, retry, dead-lettering and
@@ -91,10 +95,46 @@ export function createWorker(id, queue, options) {
         }
         finally {
             activeJobs--;
-            // Poll again immediately while there is capacity, but yield to the
-            // event loop first so a saturated queue cannot starve timers.
+            // A slot just opened: poll again immediately, yielding to the event
+            // loop first so a saturated queue cannot starve timers.
+            scheduleNextPoll(0);
+        }
+    };
+    const poll = async () => {
+        if (polling ||
+            state !== WorkerState.RUNNING ||
+            abortController?.signal.aborted) {
+            return;
+        }
+        polling = true;
+        try {
+            if (activeJobs >= concurrency) {
+                scheduleNextPoll(pollInterval);
+                return;
+            }
+            const job = await queue.claimNextJob();
+            if (!job) {
+                scheduleNextPoll(pollInterval);
+                return;
+            }
+            const proc = queue.getProcessor(job.name);
+            if (!proc) {
+                // Claimed but unrunnable: release it rather than stranding it in
+                // `active` where nothing would ever pick it up again.
+                await queue.releaseJob(job.id);
+                scheduleNextPoll(pollInterval);
+                return;
+            }
+            activeJobs++;
+            stats.processed++;
+            void runClaimedJob(job);
+            // Capacity may remain: look for more work now, not after this job
+            // settles.
             scheduleNextPoll(0);
         }
+        finally {
+            polling = false;
+        }
     };
     const clearPollTimer = () => {
         if (pollTimer) {
@@ -138,13 +178,18 @@ export function createWorker(id, queue, options) {
             }
             state = WorkerState.DRAINING;
             clearPollTimer();
-            abortController?.abort();
+            // Graceful means graceful: in-flight jobs get `drainTimeout` to
+            // finish on their own. Aborting them up front — as this once did —
+            // made `stop()` indistinguishable from `forceStop()` for any
+            // processor that honours its signal, and turned every routine
+            // shutdown into a batch of failed jobs.
             const deadline = Date.now() + Math.max(0, drainTimeout);
             while (activeJobs > 0 && Date.now() < deadline) {
                 await new Promise((resolve) => setTimeout(resolve, 25));
             }
             if (activeJobs > 0) {
                 onError(new WorkerLifecycleError(`Worker "${id}" still had ${activeJobs} job(s) in flight after ${drainTimeout}ms; forcing stop.`, { workerId: id }));
+                abortController?.abort();
             }
             clearPollTimer();
             state = WorkerState.STOPPED;

package/dist/worker/worker.type.d.ts

@@ -1,4 +1,4 @@
-import type { Queue } from "../queue/queue.type.js";
+import type { Queue, QueueLogger } from "../queue/queue.type.js";
 import type { QueueMiddleware } from "../middleware/middleware.type.js";
 import type { WorkerState } from "../jobTypes/jobTypes.type.js";
 /**
@@ -25,10 +25,13 @@ export interface WorkerOptions {
     readonly drainTimeout?: number;
     /**
      * Invoked for errors raised outside a job — a failing poll, a job that
-     * threw, or a drain that timed out. Defaults to reporting on the
-     * console. Poll errors are never left as unhandled rejections.
+     * threw, or a drain that timed out. Defaults to `logger.error`, or to
+     * `process.emitWarning` without a logger. Poll errors are never left as
+     * unhandled rejections.
      */
     readonly onError?: (error: unknown) => void;
+    /** Receives errors when no `onError` is given. */
+    readonly logger?: QueueLogger;
 }
 /**
  * Worker lifecycle states.

package/package.json

@@ -1,8 +1,12 @@
 {
   "name": "@zudojs/queue",
-  "version": "1.0.0",
+  "version": "1.2.0",
   "description": "Background job and asynchronous task infrastructure with in-memory and adapter-based queue implementations.",
   "license": "MIT",
+  "author": {
+    "name": "Oluwayemi Oyinlola",
+    "url": "https://github.com/oyinlola-tech"
+  },
   "type": "module",
   "main": "./dist/index.js",
   "module": "./dist/index.js",
@@ -21,9 +25,9 @@
     "!dist/.tsbuildinfo"
   ],
   "dependencies": {
-    "@zudojs/errors": "1.0.0",
-    "@zudojs/constants": "1.0.0",
-    "@zudojs/serialization": "1.0.0"
+    "@zudojs/errors": "1.1.0",
+    "@zudojs/constants": "1.1.0",
+    "@zudojs/serialization": "1.1.0"
   },
   "devDependencies": {
     "typescript": "7.0.2",