@zudojs/queue 0.1.0 → 1.0.0

package/dist/queue/index.d.ts

@@ -5,6 +5,6 @@
  * and managing job queues.
  */
 export { createQueue, isQueue } from "./queue.core.js";
-export type { Queue, QueueOptions, QueueStats, QueueEventMap, } from "./queue.type.js";
+export type { Queue, QueueLogger, QueueOptions, QueueStats, QueueEventMap, } from "./queue.type.js";
 export type { QueueEventEmitter } from "../queueEmitter/queueEmitter.type.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/queue/queue.core.js

@@ -1,4 +1,3 @@
-import { QueueClosedError, QueueDisposedError } from "@zudojs/errors";
 import { createInMemoryQueue } from "../inMemoryQueue/index.js";
 /**
  * Creates a new queue.
@@ -18,6 +17,7 @@ export function isQueue(value) {
         "add" in value &&
         "process" in value &&
         "getJob" in value &&
-        "getNextJob" in value);
+        "getNextJob" in value &&
+        "claimNextJob" in value);
 }
 //# sourceMappingURL=queue.core.js.map

package/dist/queue/queue.type.d.ts

@@ -5,10 +5,26 @@ import type { Processor } from "../processor/processor.type.js";
 import type { Serializer } from "../serializer/serializer.type.js";
 import type { QueueMiddleware } from "../middleware/middleware.type.js";
 import type { QueueEventEmitter } from "../queueEmitter/queueEmitter.type.js";
+import type { DeadLetterJob, DeadLetterStore } from "../deadLetter/deadLetter.type.js";
+/**
+ * Somewhere for a job to write a log line.
+ *
+ * Structurally compatible with `@zudojs/logger` and with `console`.
+ */
+export interface QueueLogger {
+    info(message: string, data?: Record<string, unknown>): void;
+}
 /**
  * Options for creating a queue.
  */
 export interface QueueOptions {
+    /**
+     * Receives lines a processor writes with `context.log()`.
+     *
+     * Without one, `JobContext.log` is a documented method that discards
+     * everything handed to it.
+     */
+    readonly logger?: QueueLogger;
     /** Queue concurrency limit. */
     readonly concurrency?: number;
     /** Default job options. */
@@ -21,6 +37,48 @@ export interface QueueOptions {
     readonly pollInterval?: number;
     /** Event emitter for queue lifecycle events. */
     readonly eventEmitter?: QueueEventEmitter;
+    /** Store that receives jobs which exhausted their attempts. */
+    readonly deadLetterStore?: DeadLetterStore<never>;
+    /**
+     * Whether `add()` rejects while the queue is paused.
+     *
+     * Defaults to `true`. Set to `false` for the conventional
+     * producer/consumer split, where pausing stops consumption only and
+     * producers may keep enqueuing.
+     */
+    readonly pauseRejectsAdd?: boolean;
+    /**
+     * Whether job payloads are round-tripped through the serializer on
+     * `add()`. Defaults to `true`, which isolates stored payloads from
+     * later caller mutation and rejects non-serializable payloads up
+     * front. Set to `false` to store payloads by reference.
+     */
+    readonly serializePayloads?: boolean;
+    /**
+     * Number of terminal (completed, failed, dead-lettered) jobs retained
+     * before the oldest are evicted. Defaults to 1000. Without a bound the
+     * queue grows for the life of the process.
+     */
+    readonly retainSettledJobs?: number;
+    /**
+     * How long `close()` waits for in-flight jobs before aborting them,
+     * in milliseconds. Defaults to 30000.
+     */
+    readonly closeTimeout?: number;
+    /**
+     * How long a job may sit in `active` without a live consumer before it
+     * is reclaimed, in milliseconds. Defaults to `0` (disabled).
+     *
+     * Guards against a consumer that claimed a job and then died, which
+     * would otherwise leave the job `active` with nothing able to retry
+     * it.
+     */
+    readonly stalledAfter?: number;
+    /**
+     * How many times a job may stall before it is dead-lettered instead of
+     * reclaimed again. Defaults to 3.
+     */
+    readonly maxStalledCount?: number;
 }
 /**
  * Statistics for a queue.
@@ -38,6 +96,16 @@ export interface QueueStats {
     readonly delayed: number;
     /** Number of retrying jobs. */
     readonly retrying: number;
+    /** Total jobs dispatched over the queue's lifetime. */
+    readonly processed: number;
+    /** Total jobs that completed successfully. */
+    readonly succeeded: number;
+    /** Total jobs that failed terminally. */
+    readonly errored: number;
+    /** Total retry attempts scheduled. */
+    readonly retried: number;
+    /** Total jobs moved to the dead letter store. */
+    readonly deadLettered: number;
 }
 /**
  * A named stream of jobs.
@@ -51,8 +119,42 @@ export interface Queue<TData = unknown> {
     process(name: string, processor: Processor<TData>): void;
     /** Get a job by ID. */
     getJob(jobId: JobId): Promise<Job<TData> | null>;
-    /** Get the next available job for processing. */
+    /**
+     * Peeks at the job that would be processed next, without claiming it.
+     *
+     * Consumers that intend to run the job must use {@link claimNextJob};
+     * peeking leaves the job `waiting`, so two consumers polling with this
+     * method would both run it.
+     */
     getNextJob(): Promise<Job<TData> | null>;
+    /**
+     * Atomically claims the next runnable job, marking it `active`.
+     *
+     * Returns `null` when nothing is runnable — including when a waiting
+     * job has no registered processor.
+     */
+    claimNextJob(): Promise<Job<TData> | null>;
+    /**
+     * Returns a claimed job to the waiting pool.
+     *
+     * Used when a consumer claims a job it turns out it cannot run, so the
+     * job is retried by another consumer rather than stranded in `active`.
+     */
+    releaseJob(jobId: JobId): Promise<boolean>;
+    /**
+     * Runs an already-claimed job through the queue's processing pipeline.
+     *
+     * The queue owns job state, so consumers must dispatch through this
+     * rather than invoking a processor themselves — doing so would leave
+     * the job stuck in `active` and skip retry, dead-lettering and
+     * middleware.
+     */
+    runJob(job: Job<TData>, options?: {
+        middleware?: readonly QueueMiddleware[];
+        signal?: AbortSignal;
+        /** Fallback timeout for a job that carries none of its own. */
+        timeoutMs?: number;
+    }): Promise<void>;
     /** Get a registered processor by job name. */
     getProcessor(name: string): Processor<TData> | undefined;
     /** Get queue statistics. */
@@ -63,7 +165,11 @@ export interface Queue<TData = unknown> {
     resume(): Promise<void>;
     /** Check if the queue is paused. */
     isPaused(): boolean;
-    /** Close the queue. */
+    /** Check if the queue has been closed. */
+    isDisposed(): boolean;
+    /** Get the jobs that exhausted their attempts. */
+    getDeadLetterJobs(): Promise<readonly DeadLetterJob<TData>[]>;
+    /** Close the queue, draining in-flight jobs first. */
     close(): Promise<void>;
 }
 /**

package/dist/queueEmitter/queueEmitter.core.d.ts

@@ -2,20 +2,40 @@ import type { QueueEventEmitter } from "./queueEmitter.type.js";
 import type { QueueEventMap } from "../queue/queue.type.js";
 type EventName = keyof QueueEventMap;
 type Handler<T extends EventName> = (data: QueueEventMap[T]) => void;
+/**
+ * Options for the in-memory queue event emitter.
+ */
+export interface QueueEventEmitterOptions {
+    /**
+     * Invoked when a handler throws. Defaults to reporting the error on the
+     * next tick so it surfaces in normal error reporting without unwinding
+     * the emitting code path.
+     */
+    readonly onHandlerError?: (error: unknown, event: EventName) => void;
+}
 /**
  * In-memory queue event emitter.
  *
- * Stores handlers in memory and emits events synchronously.
+ * Stores handlers in memory and emits events synchronously. A handler
+ * that throws is isolated: the remaining handlers still run, and the
+ * failure never propagates back into queue processing, where it would be
+ * misreported as a job failure.
  */
 export declare class InMemoryQueueEventEmitter implements QueueEventEmitter {
     private readonly handlers;
+    private readonly onHandlerError;
+    constructor(options?: QueueEventEmitterOptions);
     emit<K extends EventName>(event: K, data: QueueEventMap[K]): void;
     on<K extends EventName>(event: K, handler: Handler<K>): () => void;
+    /**
+     * Removes every registered handler.
+     */
+    removeAllListeners(): void;
 }
 /**
  * Creates an in-memory queue event emitter.
  */
-export declare function createInMemoryQueueEventEmitter(): QueueEventEmitter;
+export declare function createInMemoryQueueEventEmitter(options?: QueueEventEmitterOptions): QueueEventEmitter;
 /**
  * Creates a no-op queue event emitter.
  */

package/dist/queueEmitter/queueEmitter.core.js

@@ -1,16 +1,37 @@
 /**
  * In-memory queue event emitter.
  *
- * Stores handlers in memory and emits events synchronously.
+ * Stores handlers in memory and emits events synchronously. A handler
+ * that throws is isolated: the remaining handlers still run, and the
+ * failure never propagates back into queue processing, where it would be
+ * misreported as a job failure.
  */
 export class InMemoryQueueEventEmitter {
     handlers = new Map();
+    onHandlerError;
+    constructor(options = {}) {
+        this.onHandlerError =
+            options.onHandlerError ??
+                ((error, event) => {
+                    queueMicrotask(() => {
+                        console.error(`[@zudojs/queue] Listener for "${event}" threw.`, error);
+                    });
+                });
+    }
     emit(event, data) {
         const handlers = this.handlers.get(event);
-        if (handlers) {
-            for (const handler of handlers) {
+        if (!handlers) {
+            return;
+        }
+        // Snapshot so a handler that subscribes or unsubscribes during
+        // dispatch cannot mutate the set being iterated.
+        for (const handler of [...handlers]) {
+            try {
                 handler(data);
             }
+            catch (error) {
+                this.onHandlerError(error, event);
+            }
         }
     }
     on(event, handler) {
@@ -19,17 +40,36 @@ export class InMemoryQueueEventEmitter {
             handlers = new Set();
             this.handlers.set(event, handlers);
         }
-        handlers.add(handler);
+        const registered = handler;
+        handlers.add(registered);
+        let removed = false;
         return () => {
-            handlers.delete(handler);
+            if (removed) {
+                return;
+            }
+            removed = true;
+            const current = this.handlers.get(event);
+            if (!current) {
+                return;
+            }
+            current.delete(registered);
+            if (current.size === 0) {
+                this.handlers.delete(event);
+            }
         };
     }
+    /**
+     * Removes every registered handler.
+     */
+    removeAllListeners() {
+        this.handlers.clear();
+    }
 }
 /**
  * Creates an in-memory queue event emitter.
  */
-export function createInMemoryQueueEventEmitter() {
-    return new InMemoryQueueEventEmitter();
+export function createInMemoryQueueEventEmitter(options) {
+    return new InMemoryQueueEventEmitter(options);
 }
 /**
  * Creates a no-op queue event emitter.

package/dist/queueManager/queueManager.core.js

@@ -1,3 +1,4 @@
+import { QueueError } from "@zudojs/errors";
 import { createQueue } from "../queue/queue.core.js";
 /**
  * Creates a new QueueManager.
@@ -5,12 +6,23 @@ import { createQueue } from "../queue/queue.core.js";
 export function createQueueManager() {
     const queues = new Map();
     return {
+        /**
+         * Returns the queue with this name, creating it on first request.
+         *
+         * `options` apply only when the queue is created. Passing different
+         * options for an existing queue throws rather than silently
+         * returning a queue configured some other way.
+         */
         getQueue(name, options) {
-            let queue = queues.get(name);
-            if (!queue) {
-                queue = createQueue(name, options);
-                queues.set(name, queue);
+            const existing = queues.get(name);
+            if (existing) {
+                if (options !== undefined) {
+                    throw new QueueError(`Queue "${name}" already exists; options are only applied when a queue is created.`, { queueName: name });
+                }
+                return existing;
             }
+            const queue = createQueue(name, options);
+            queues.set(name, queue);
             return queue;
         },
         getExistingQueue(name) {
@@ -22,10 +34,17 @@ export function createQueueManager() {
         getQueueNames() {
             return Array.from(queues.keys());
         },
+        /**
+         * Closes every queue, clearing the registry even if some fail.
+         */
         async closeAll() {
             const closePromises = Array.from(queues.values()).map((queue) => queue.close());
-            await Promise.all(closePromises);
             queues.clear();
+            const results = await Promise.allSettled(closePromises);
+            const failure = results.find((result) => result.status === "rejected");
+            if (failure && failure.status === "rejected") {
+                throw failure.reason;
+            }
         },
     };
 }

package/dist/queueRegistry/queueRegistry.core.js

@@ -28,11 +28,26 @@ export function createQueueRegistry() {
             return Array.from(infoMap.values());
         },
         unregister(name) {
+            const queue = queues.get(name);
             const deleted = queues.delete(name);
             infoMap.delete(name);
+            // Dropping the reference without closing would leave the queue's
+            // poll timer running with nothing able to stop it.
+            if (queue) {
+                void queue.close().catch(() => { });
+            }
             return deleted;
         },
+        async closeAll() {
+            const pending = Array.from(queues.values()).map((queue) => queue.close());
+            queues.clear();
+            infoMap.clear();
+            await Promise.allSettled(pending);
+        },
         clear() {
+            for (const queue of queues.values()) {
+                void queue.close().catch(() => { });
+            }
             queues.clear();
             infoMap.clear();
         },

package/dist/queueRegistry/queueRegistry.type.d.ts

@@ -15,6 +15,8 @@ export interface QueueInfo {
  * Registry for queue instances.
  */
 export interface QueueRegistry {
+    /** Close every registered queue and clear the registry. */
+    closeAll(): Promise<void>;
     /** Register a queue. */
     register<TData>(queue: Queue<TData>): void;
     /** Get a queue by name. */

package/dist/retryPolicy/retryPolicy.core.d.ts

@@ -1,9 +1,44 @@
-import { BackoffType } from "../jobTypes/jobTypes.type.js";
+import type { BackoffStrategy } from "../jobTypes/jobTypes.type.js";
 import type { BackoffOptions } from "../jobOptions/jobOptions.type.js";
+/**
+ * Largest delay Node's timer subsystem accepts.
+ *
+ * A `setTimeout` delay above this overflows a signed 32-bit integer; Node
+ * warns and clamps it to `1`, turning a long backoff into an immediate
+ * retry. Every computed delay is clamped to this ceiling instead.
+ */
+export declare const MAX_TIMER_DELAY = 2147483647;
+/**
+ * Backoff applied to a retryable job that declares no backoff of its own.
+ *
+ * Without this a job configured with `attempts > 1` and no `backoff` would
+ * retry with a zero delay, hammering whatever dependency just failed.
+ */
+export declare const DEFAULT_RETRY_BACKOFF: BackoffOptions;
+/**
+ * Applies the configured jitter strategy to a computed delay.
+ *
+ * `full` spreads retries uniformly across `[0, delay]`; `equal` keeps half
+ * the delay fixed and randomises the other half. Both break up the
+ * synchronised retry waves that a purely deterministic backoff produces
+ * when many jobs fail against the same dependency at the same moment.
+ */
+export declare function applyJitter(delay: number, jitter: BackoffOptions["jitter"], random?: () => number): number;
 /**
  * Calculates the delay for the next retry attempt.
+ *
+ * The result is always a finite, non-negative number no greater than
+ * {@link MAX_TIMER_DELAY}. Jitter is applied only when the backoff opts
+ * into it, so a backoff without `jitter` stays deterministic.
  */
 export declare function calculateRetryDelay(attempt: number, backoff?: BackoffOptions): number;
+/**
+ * Resolves the backoff to use for a job.
+ *
+ * Falls back to {@link DEFAULT_RETRY_BACKOFF} so that a retryable job
+ * never retries with a zero delay.
+ */
+export declare function resolveBackoff(backoff?: BackoffOptions): BackoffOptions;
 /**
  * Checks if a job should be retried based on its state.
  */
@@ -11,19 +46,23 @@ export declare function shouldRetry(attempt: number, maxAttempts: number): boole
 /**
  * Creates a backoff options object.
  */
-export declare function createBackoffOptions(type: BackoffType, delay: number, options?: {
+export declare function createBackoffOptions(type: BackoffStrategy, delay: number, options?: {
     maxDelay?: number;
     multiplier?: number;
+    jitter?: BackoffOptions["jitter"];
 }): BackoffOptions;
 /**
  * Creates a fixed backoff options.
  */
-export declare function createFixedBackoff(delay: number): BackoffOptions;
+export declare function createFixedBackoff(delay: number, options?: {
+    jitter?: BackoffOptions["jitter"];
+}): BackoffOptions;
 /**
  * Creates an exponential backoff options.
  */
 export declare function createExponentialBackoff(delay: number, options?: {
     maxDelay?: number;
     multiplier?: number;
+    jitter?: BackoffOptions["jitter"];
 }): BackoffOptions;
 //# sourceMappingURL=retryPolicy.core.d.ts.map

package/dist/retryPolicy/retryPolicy.core.js

@@ -1,23 +1,91 @@
 import { BackoffType } from "../jobTypes/jobTypes.type.js";
+/**
+ * Largest delay Node's timer subsystem accepts.
+ *
+ * A `setTimeout` delay above this overflows a signed 32-bit integer; Node
+ * warns and clamps it to `1`, turning a long backoff into an immediate
+ * retry. Every computed delay is clamped to this ceiling instead.
+ */
+export const MAX_TIMER_DELAY = 2_147_483_647;
+/**
+ * Backoff applied to a retryable job that declares no backoff of its own.
+ *
+ * Without this a job configured with `attempts > 1` and no `backoff` would
+ * retry with a zero delay, hammering whatever dependency just failed.
+ */
+export const DEFAULT_RETRY_BACKOFF = Object.freeze({
+    type: BackoffType.EXPONENTIAL,
+    delay: 1_000,
+    maxDelay: 30_000,
+    multiplier: 2,
+    jitter: "full",
+});
+/**
+ * Clamps a delay to the range a timer can actually represent.
+ */
+function clampDelay(delay) {
+    if (!Number.isFinite(delay) || delay < 0) {
+        return 0;
+    }
+    return Math.min(delay, MAX_TIMER_DELAY);
+}
+/**
+ * Applies the configured jitter strategy to a computed delay.
+ *
+ * `full` spreads retries uniformly across `[0, delay]`; `equal` keeps half
+ * the delay fixed and randomises the other half. Both break up the
+ * synchronised retry waves that a purely deterministic backoff produces
+ * when many jobs fail against the same dependency at the same moment.
+ */
+export function applyJitter(delay, jitter, random = Math.random) {
+    if (delay <= 0) {
+        return 0;
+    }
+    switch (jitter) {
+        case "full":
+            return Math.round(random() * delay);
+        case "equal":
+            return Math.round(delay / 2 + random() * (delay / 2));
+        default:
+            return delay;
+    }
+}
 /**
  * Calculates the delay for the next retry attempt.
+ *
+ * The result is always a finite, non-negative number no greater than
+ * {@link MAX_TIMER_DELAY}. Jitter is applied only when the backoff opts
+ * into it, so a backoff without `jitter` stays deterministic.
  */
 export function calculateRetryDelay(attempt, backoff) {
     if (!backoff) {
         return 0;
     }
-    const { type, delay, maxDelay, multiplier } = backoff;
-    switch (type) {
-        case BackoffType.FIXED:
-            return delay;
-        case BackoffType.EXPONENTIAL: {
-            const multiplierValue = multiplier ?? 2;
-            const calculatedDelay = delay * Math.pow(multiplierValue, attempt - 1);
-            return maxDelay ? Math.min(calculatedDelay, maxDelay) : calculatedDelay;
+    const { type, delay, maxDelay, multiplier, jitter } = backoff;
+    const base = () => {
+        switch (type) {
+            case BackoffType.FIXED:
+                return delay;
+            case BackoffType.EXPONENTIAL: {
+                const multiplierValue = multiplier ?? 2;
+                const exponent = Math.max(0, attempt - 1);
+                const calculatedDelay = delay * Math.pow(multiplierValue, exponent);
+                return maxDelay ? Math.min(calculatedDelay, maxDelay) : calculatedDelay;
+            }
+            default:
+                return delay;
         }
-        default:
-            return delay;
-    }
+    };
+    return clampDelay(applyJitter(clampDelay(base()), jitter));
+}
+/**
+ * Resolves the backoff to use for a job.
+ *
+ * Falls back to {@link DEFAULT_RETRY_BACKOFF} so that a retryable job
+ * never retries with a zero delay.
+ */
+export function resolveBackoff(backoff) {
+    return backoff ?? DEFAULT_RETRY_BACKOFF;
 }
 /**
  * Checks if a job should be retried based on its state.
@@ -34,13 +102,14 @@ export function createBackoffOptions(type, delay, options) {
         delay,
         maxDelay: options?.maxDelay,
         multiplier: options?.multiplier,
+        jitter: options?.jitter,
     };
 }
 /**
  * Creates a fixed backoff options.
  */
-export function createFixedBackoff(delay) {
-    return createBackoffOptions(BackoffType.FIXED, delay);
+export function createFixedBackoff(delay, options) {
+    return createBackoffOptions(BackoffType.FIXED, delay, options);
 }
 /**
  * Creates an exponential backoff options.

package/dist/serializer/serializer.core.d.ts

@@ -10,8 +10,9 @@ export declare const JsonSerializer: Serializer;
  * Creates a serializer with custom options.
  */
 export declare function createJsonSerializer(options?: {
-    replacer?: (key: string, value: unknown) => unknown;
+    /** Indentation width. Any value turns on pretty-printing. */
     space?: number;
+    /** Preserve BigInt, Date, Map, Set and Uint8Array across the round trip. */
     preserveTypes?: boolean;
 }): Serializer;
 /** No-op serializer that passes data through unchanged. */

package/dist/worker/worker.core.d.ts

@@ -2,6 +2,10 @@ import type { Queue } from "../queue/queue.type.js";
 import type { Worker, WorkerOptions } from "./worker.type.js";
 /**
  * Creates a new Worker.
+ *
+ * The worker claims each job before running it, so a job is never picked
+ * up twice — by this worker on its next poll, or by another worker on the
+ * same queue.
  */
 export declare function createWorker<TData>(id: string, queue: Queue<TData>, options?: WorkerOptions): Worker<TData>;
 /**