@zudojs/queue 1.2.0 → 1.3.0

package/README.md

@@ -69,8 +69,8 @@ await queue.close();
 - Draining shutdown with a bounded `closeTimeout`, then cooperative abort
 - Stalled-job reclaim (`stalledAfter`, `maxStalledCount`) for consumers that die
   mid-job
-- Bounded retention of settled jobs, so a long-lived queue does not grow without
-  limit
+- Bounded retention of settled jobs and of dead-lettered jobs, so a long-lived
+  queue does not grow without limit
 
 ## Workers, timeouts and context
 
@@ -120,6 +120,10 @@ Keep captured values small and serializable (ids, not live objects). With
 `@zudojs/tenancy`, capture the tenant id and restore it with the tenant context
 storage's `run`.
 
+The `zudo:context` metadata key is reserved: it is written only by the queue's
+own carriers and is stripped from any `metadata` passed to `add()`, so an
+enqueuer cannot choose the context its job runs under.
+
 Errors with no caller to receive them (a worker's failing poll, a throwing event
 listener) go to `logger.error` when a logger is configured, and otherwise to
 `process.emitWarning`, never to `console`.

package/dist/contextCarrier/contextCarrier.core.d.ts

@@ -13,10 +13,15 @@ export declare const CONTEXT_METADATA_KEY = "zudo:context";
 /**
  * Captures every carrier's value into a metadata record.
  *
+ * {@link CONTEXT_METADATA_KEY} is owned by the queue: a value the caller put
+ * under that key is always discarded, whether or not a carrier captured
+ * anything, so an enqueuer cannot forge the tenant, correlation id or trace
+ * the processor runs under.
+ *
  * @param carriers - The queue's context carriers.
  * @param metadata - The job's own metadata, if any.
  * @returns The metadata with a {@link CONTEXT_METADATA_KEY} entry added, or
- *   the input unchanged when no carrier captured anything.
+ *   without one when no carrier captured anything.
  */
 export declare function captureContext(carriers: readonly QueueContextCarrier[] | undefined, metadata: Record<string, unknown> | undefined): Record<string, unknown> | undefined;
 /**

package/dist/contextCarrier/contextCarrier.core.js

@@ -8,17 +8,37 @@
  * `{ [carrier.key]: value }` record.
  */
 export const CONTEXT_METADATA_KEY = "zudo:context";
+/**
+ * Strips the reserved context key from caller-supplied metadata.
+ *
+ * The key is owned by the queue on every path: whatever the enqueuer put
+ * there is dropped, so `add()` metadata can never supply a context record of
+ * its own. Returns the input untouched when there is nothing to strip, so the
+ * common case still allocates nothing.
+ */
+function withoutStoredContext(metadata) {
+    if (!metadata || !Object.hasOwn(metadata, CONTEXT_METADATA_KEY)) {
+        return metadata;
+    }
+    const { [CONTEXT_METADATA_KEY]: _discarded, ...rest } = metadata;
+    return rest;
+}
 /**
  * Captures every carrier's value into a metadata record.
  *
+ * {@link CONTEXT_METADATA_KEY} is owned by the queue: a value the caller put
+ * under that key is always discarded, whether or not a carrier captured
+ * anything, so an enqueuer cannot forge the tenant, correlation id or trace
+ * the processor runs under.
+ *
  * @param carriers - The queue's context carriers.
  * @param metadata - The job's own metadata, if any.
  * @returns The metadata with a {@link CONTEXT_METADATA_KEY} entry added, or
- *   the input unchanged when no carrier captured anything.
+ *   without one when no carrier captured anything.
  */
 export function captureContext(carriers, metadata) {
     if (!carriers || carriers.length === 0)
-        return metadata;
+        return withoutStoredContext(metadata);
     const captured = {};
     let any = false;
     for (const carrier of carriers) {
@@ -29,7 +49,7 @@ export function captureContext(carriers, metadata) {
         any = true;
     }
     if (!any)
-        return metadata;
+        return withoutStoredContext(metadata);
     return { ...metadata, [CONTEXT_METADATA_KEY]: Object.freeze(captured) };
 }
 /**

package/dist/deadLetter/deadLetter.core.d.ts

@@ -1,9 +1,34 @@
 import type { Job } from "../job/job.type.js";
 import type { DeadLetterStore } from "./deadLetter.type.js";
+/**
+ * Dead-lettered jobs retained by the in-memory store when no cap is given.
+ *
+ * Mirrors the queue's own `DEFAULT_RETAINED_JOBS`: the dead letter store
+ * keeps a full copy of every failed job — payload, metadata and error — so an
+ * unbounded one grows for the life of the process at whatever the failure
+ * rate happens to be.
+ */
+export declare const DEFAULT_DEAD_LETTER_JOBS = 1000;
+/**
+ * Options for {@link createInMemoryDeadLetterStore}.
+ */
+export interface InMemoryDeadLetterStoreOptions {
+    /**
+     * Maximum number of dead-lettered jobs retained before the oldest are
+     * evicted. Defaults to {@link DEFAULT_DEAD_LETTER_JOBS}. Pass
+     * `Number.POSITIVE_INFINITY` for the previous unbounded behaviour.
+     */
+    readonly maxEntries?: number;
+}
 /**
  * Creates an in-memory dead letter store.
+ *
+ * Retention is bounded: past `maxEntries` the oldest entry is evicted, so a
+ * long-lived queue with a steady failure rate does not grow without limit.
+ *
+ * @param options - Retention options.
  */
-export declare function createInMemoryDeadLetterStore<TData = unknown>(): DeadLetterStore<TData>;
+export declare function createInMemoryDeadLetterStore<TData = unknown>(options?: InMemoryDeadLetterStoreOptions): DeadLetterStore<TData>;
 /**
  * Moves a failed job to the dead letter store.
  */

package/dist/deadLetter/deadLetter.core.js

@@ -1,12 +1,38 @@
 import { toQueueError } from "@zudojs/errors";
+/**
+ * Dead-lettered jobs retained by the in-memory store when no cap is given.
+ *
+ * Mirrors the queue's own `DEFAULT_RETAINED_JOBS`: the dead letter store
+ * keeps a full copy of every failed job — payload, metadata and error — so an
+ * unbounded one grows for the life of the process at whatever the failure
+ * rate happens to be.
+ */
+export const DEFAULT_DEAD_LETTER_JOBS = 1_000;
 /**
  * Creates an in-memory dead letter store.
+ *
+ * Retention is bounded: past `maxEntries` the oldest entry is evicted, so a
+ * long-lived queue with a steady failure rate does not grow without limit.
+ *
+ * @param options - Retention options.
  */
-export function createInMemoryDeadLetterStore() {
+export function createInMemoryDeadLetterStore(options) {
     const store = new Map();
+    const configured = options?.maxEntries ?? DEFAULT_DEAD_LETTER_JOBS;
+    const maxEntries = configured > 0 ? configured : DEFAULT_DEAD_LETTER_JOBS;
     return {
         async add(deadLetterJob) {
+            // Re-inserting an existing id must not count as a new entry, so
+            // delete first: a Map keeps its original insertion order otherwise,
+            // which would also evict the wrong entry.
+            store.delete(deadLetterJob.job.id);
             store.set(deadLetterJob.job.id, deadLetterJob);
+            while (store.size > maxEntries) {
+                const oldest = store.keys().next().value;
+                if (oldest === undefined)
+                    break;
+                store.delete(oldest);
+            }
         },
         async get(jobId) {
             return store.get(jobId) ?? null;

package/dist/deadLetter/index.d.ts

@@ -4,6 +4,7 @@
  * Provides storage and management for jobs that have
  * exceeded their retry attempts.
  */
-export { createInMemoryDeadLetterStore, moveToDeadLetter, } from "./deadLetter.core.js";
+export { DEFAULT_DEAD_LETTER_JOBS, createInMemoryDeadLetterStore, moveToDeadLetter, } from "./deadLetter.core.js";
+export type { InMemoryDeadLetterStoreOptions } from "./deadLetter.core.js";
 export type { DeadLetterJob, DeadLetterStore } from "./deadLetter.type.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/deadLetter/index.js

@@ -4,5 +4,5 @@
  * Provides storage and management for jobs that have
  * exceeded their retry attempts.
  */
-export { createInMemoryDeadLetterStore, moveToDeadLetter, } from "./deadLetter.core.js";
+export { DEFAULT_DEAD_LETTER_JOBS, createInMemoryDeadLetterStore, moveToDeadLetter, } from "./deadLetter.core.js";
 //# sourceMappingURL=index.js.map

package/dist/inMemoryQueue/inMemoryQueue.core.d.ts

@@ -4,6 +4,7 @@ import type { Queue, QueueOptions, QueueStats } from "../queue/queue.type.js";
 import type { Processor } from "../processor/processor.type.js";
 import type { JobOptions } from "../jobOptions/jobOptions.type.js";
 import type { QueueMiddleware } from "../middleware/middleware.type.js";
+import type { QueueEventEmitter } from "../queueEmitter/queueEmitter.type.js";
 import type { DeadLetterJob } from "../deadLetter/deadLetter.type.js";
 /**
  * In-memory queue implementation.
@@ -32,6 +33,11 @@ export declare class InMemoryQueue<TData = unknown> implements Queue<TData> {
     private readonly stalledCounts;
     private readonly deduplicationIndex;
     private readonly deadLetterStore;
+    /**
+     * Whether this queue created its own dead letter store. A store handed in
+     * by the caller outlives the queue and is theirs to clear.
+     */
+    private readonly ownsDeadLetterStore;
     private readonly emitter;
     private readonly counters;
     private emptySince;
@@ -46,6 +52,13 @@ export declare class InMemoryQueue<TData = unknown> implements Queue<TData> {
      * concurrency apply to every job.
      */
     setAutoProcess(enabled: boolean): void;
+    /**
+     * The emitter this queue publishes lifecycle events on.
+     *
+     * A no-op emitter when the queue was created without one, so a worker can
+     * report its lifecycle unconditionally.
+     */
+    get events(): QueueEventEmitter;
     add(jobName: string, data: TData, options?: JobOptions): Promise<Job<TData>>;
     process(name: string, processor: Processor<TData>): void;
     getJob(jobId: JobId): Promise<Job<TData> | null>;

package/dist/inMemoryQueue/inMemoryQueue.core.js

@@ -4,8 +4,8 @@ import { assertProcessor } from "../processor/processor.type.js";
 import { createJob, updateJobState } from "../job/job.core.js";
 import { JobState as JobStateEnum, createJobName, } from "../jobTypes/jobTypes.type.js";
 import { JsonSerializer } from "../serializer/serializer.core.js";
-import { createInMemoryDeadLetterStore } from "../deadLetter/deadLetter.core.js";
-import { createNoopQueueEventEmitter } from "../queueEmitter/queueEmitter.core.js";
+import { DEFAULT_DEAD_LETTER_JOBS, createInMemoryDeadLetterStore, } from "../deadLetter/deadLetter.core.js";
+import { InMemoryQueueEventEmitter, createNoopQueueEventEmitter, } from "../queueEmitter/queueEmitter.core.js";
 import { processJob } from "./inMemoryQueue.processing.js";
 import { captureContext } from "../contextCarrier/contextCarrier.core.js";
 import { scheduleJob, promoteDueScheduledJobs, } from "./inMemoryQueue.scheduling.js";
@@ -47,6 +47,11 @@ export class InMemoryQueue {
     stalledCounts = new Map();
     deduplicationIndex = new Map();
     deadLetterStore;
+    /**
+     * Whether this queue created its own dead letter store. A store handed in
+     * by the caller outlives the queue and is theirs to clear.
+     */
+    ownsDeadLetterStore;
     emitter;
     counters = {
         processedCount: 0,
@@ -63,8 +68,19 @@ export class InMemoryQueue {
         this.serializer = this.options.serializer ?? JsonSerializer;
         this.middleware = this.options.middleware ?? [];
         this.emitter = options?.eventEmitter ?? createNoopQueueEventEmitter();
+        // A supplied emitter is built before the queue exists, so it cannot have
+        // been given the queue's logger. Hand it over, so a throwing listener is
+        // reported through structured logging rather than `process.emitWarning`.
+        if (this.options.logger &&
+            this.emitter instanceof InMemoryQueueEventEmitter) {
+            this.emitter.setLogger(this.options.logger);
+        }
+        this.ownsDeadLetterStore = this.options.deadLetterStore === undefined;
         this.deadLetterStore =
-            this.options.deadLetterStore ?? createInMemoryDeadLetterStore();
+            this.options.deadLetterStore ??
+                createInMemoryDeadLetterStore({
+                    maxEntries: DEFAULT_DEAD_LETTER_JOBS,
+                });
         this.autoProcess = this.options.autoProcess ?? true;
     }
     /**
@@ -78,6 +94,15 @@ export class InMemoryQueue {
     setAutoProcess(enabled) {
         this.autoProcess = enabled;
     }
+    /**
+     * The emitter this queue publishes lifecycle events on.
+     *
+     * A no-op emitter when the queue was created without one, so a worker can
+     * report its lifecycle unconditionally.
+     */
+    get events() {
+        return this.emitter;
+    }
     async add(jobName, data, options) {
         if (this.disposed)
             throw new QueueDisposedError(this.name);
@@ -277,6 +302,11 @@ export class InMemoryQueue {
         this.settledOrder.length = 0;
         this.stalledCounts.clear();
         this.inFlight.clear();
+        // A dead letter store the queue created dies with it; one handed in by
+        // the caller is theirs and is left alone.
+        if (this.ownsDeadLetterStore) {
+            await this.deadLetterStore.clear();
+        }
         this.activeCount = 0;
         this.paused = false;
         this.emptySince = 0;
@@ -400,17 +430,23 @@ export class InMemoryQueue {
         }
         const abortController = new AbortController();
         // A consumer's own signal (a worker draining, say) must reach the
-        // job it dispatched.
-        if (options?.signal) {
-            if (options.signal.aborted) {
-                abortController.abort(options.signal.reason);
+        // job it dispatched. The forwarder is held so it can be removed once
+        // the job settles: a worker uses one long-lived signal for every job
+        // it dispatches, so a listener left behind accumulates for the life
+        // of the worker and pins that job's controller with it.
+        const consumerSignal = options?.signal;
+        let forwardAbort;
+        if (consumerSignal) {
+            if (consumerSignal.aborted) {
+                abortController.abort(consumerSignal.reason);
             }
             else {
-                options.signal.addEventListener("abort", () => {
+                forwardAbort = () => {
                     if (!abortController.signal.aborted) {
-                        abortController.abort(options.signal?.reason);
+                        abortController.abort(consumerSignal.reason);
                     }
-                }, { once: true });
+                };
+                consumerSignal.addEventListener("abort", forwardAbort, { once: true });
             }
         }
         this.inFlight.set(job.id, abortController);
@@ -453,6 +489,9 @@ export class InMemoryQueue {
         finally {
             this.activeCount--;
             this.inFlight.delete(job.id);
+            if (forwardAbort && consumerSignal) {
+                consumerSignal.removeEventListener("abort", forwardAbort);
+            }
         }
     }
     async processTick() {

package/dist/inMemoryQueue/inMemoryQueue.processing.js

@@ -48,10 +48,15 @@ export async function processJob(job, processor, options, deps) {
         },
     });
     const timeoutMs = options.timeoutMs ?? DEFAULT_JOB_OPTIONS.timeout ?? 30_000;
+    // Whether this job's own timeout is what aborted it. An abort from
+    // anywhere else — a draining worker, `close()`, a consumer's signal — is a
+    // cancellation, and is reported as one.
+    let timedOut = false;
     const timeoutMiddleware = createTimeoutMiddleware(timeoutMs, () => {
         // Let a cooperative processor observe the timeout and stop working
         // instead of running on with its result discarded.
         if (!abortController.signal.aborted) {
+            timedOut = true;
             abortController.abort(new Error(`Job "${updatedJob.id}" timed out after ${timeoutMs}ms.`));
         }
     });
@@ -92,6 +97,9 @@ export async function processJob(job, processor, options, deps) {
     catch (error) {
         const errorMessage = error instanceof Error ? error.message : String(error);
         await settleWithin(running, options.timeoutGraceMs ?? DEFAULT_TIMEOUT_GRACE_MS);
+        if (abortController.signal.aborted && !timedOut) {
+            emitter.emit("job:cancelled", { job: updatedJob });
+        }
         await handleJobFailure(updatedJob, errorMessage, deps);
     }
     finally {

package/dist/queue/queue.type.d.ts

@@ -138,6 +138,14 @@ export interface QueueStats {
 export interface Queue<TData = unknown> {
     /** Queue name. */
     readonly name: QueueName;
+    /**
+     * The emitter this queue publishes lifecycle events on, when it has one.
+     *
+     * Exposed so a `Worker` can report its own lifecycle
+     * (`worker:started`, `worker:stopped`, `worker:error`) on the same
+     * emitter as the jobs it runs.
+     */
+    readonly events?: QueueEventEmitter;
     /** Add a job to the queue. */
     add(name: string, data: TData, options?: JobOptions): Promise<Job<TData>>;
     /** Process jobs with a processor. */
@@ -228,6 +236,7 @@ export type QueueEventMap = {
         job: Job;
         attempt: number;
     };
+    /** A running job was aborted from outside — a drain, a close, a cancel. */
     "job:cancelled": {
         job: Job;
     };

package/dist/queueEmitter/queueEmitter.core.d.ts

@@ -1,5 +1,5 @@
 import type { QueueEventEmitter } from "./queueEmitter.type.js";
-import type { QueueEventMap } from "../queue/queue.type.js";
+import type { QueueEventMap, QueueLogger } from "../queue/queue.type.js";
 type EventName = keyof QueueEventMap;
 type Handler<T extends EventName> = (data: QueueEventMap[T]) => void;
 /**
@@ -12,6 +12,12 @@ export interface QueueEventEmitterOptions {
      * the emitting code path.
      */
     readonly onHandlerError?: (error: unknown, event: EventName) => void;
+    /**
+     * Receives a throwing listener's error when no `onHandlerError` is given.
+     * Without one the failure goes to `process.emitWarning`, bypassing
+     * structured logging and redaction.
+     */
+    readonly logger?: QueueLogger;
 }
 /**
  * In-memory queue event emitter.
@@ -24,7 +30,20 @@ export interface QueueEventEmitterOptions {
 export declare class InMemoryQueueEventEmitter implements QueueEventEmitter {
     private readonly handlers;
     private readonly onHandlerError;
+    /** Whether `onHandlerError` was supplied, so `setLogger` leaves it alone. */
+    private readonly hasCustomHandlerError;
+    private logger;
     constructor(options?: QueueEventEmitterOptions);
+    /**
+     * Adopts a logger for the default handler-error report.
+     *
+     * Called by a queue that was configured with a logger, since the emitter is
+     * built before the queue exists and cannot have been given it. A logger or
+     * an `onHandlerError` supplied at construction always wins.
+     *
+     * @param logger - Destination for a throwing listener's error.
+     */
+    setLogger(logger: QueueLogger): void;
     emit<K extends EventName>(event: K, data: QueueEventMap[K]): void;
     on<K extends EventName>(event: K, handler: Handler<K>): () => void;
     /**

package/dist/queueEmitter/queueEmitter.core.js

@@ -10,15 +10,34 @@ import { reportQueueError } from "../queue/queue.report.js";
 export class InMemoryQueueEventEmitter {
     handlers = new Map();
     onHandlerError;
+    /** Whether `onHandlerError` was supplied, so `setLogger` leaves it alone. */
+    hasCustomHandlerError;
+    logger;
     constructor(options = {}) {
+        this.hasCustomHandlerError = options.onHandlerError !== undefined;
+        this.logger = options.logger;
         this.onHandlerError =
             options.onHandlerError ??
                 ((error, event) => {
                     queueMicrotask(() => {
-                        reportQueueError(`[@zudojs/queue] Listener for "${event}" threw.`, error);
+                        reportQueueError(`[@zudojs/queue] Listener for "${event}" threw.`, error, this.logger);
                     });
                 });
     }
+    /**
+     * Adopts a logger for the default handler-error report.
+     *
+     * Called by a queue that was configured with a logger, since the emitter is
+     * built before the queue exists and cannot have been given it. A logger or
+     * an `onHandlerError` supplied at construction always wins.
+     *
+     * @param logger - Destination for a throwing listener's error.
+     */
+    setLogger(logger) {
+        if (this.hasCustomHandlerError || this.logger)
+            return;
+        this.logger = logger;
+    }
     emit(event, data) {
         const handlers = this.handlers.get(event);
         if (!handlers) {

package/dist/worker/worker.core.js

@@ -26,12 +26,29 @@ export function createWorker(id, queue, options) {
     let polling = false;
     let abortController = null;
     queue.setAutoProcess?.(false);
-    const onError = options?.onError ??
+    /**
+     * Reports the worker's own lifecycle on the queue's emitter.
+     *
+     * `worker:started`, `worker:stopped` and `worker:error` were part of
+     * `QueueEventMap` with nothing emitting them, so a consumer subscribing for
+     * readiness never heard from the worker.
+     */
+    const emitLifecycle = (event) => {
+        queue.events?.emit(event, { workerId: id });
+    };
+    const reportWorkerError = options?.onError ??
         ((error) => {
             queueMicrotask(() => {
                 reportQueueError(`[@zudojs/queue] Worker "${id}" poll failed.`, error, options?.logger);
             });
         });
+    const onError = (error) => {
+        queue.events?.emit("worker:error", {
+            workerId: id,
+            error: error instanceof Error ? error : new Error(String(error)),
+        });
+        reportWorkerError(error);
+    };
     /**
      * Arms the next poll. At most one timer is ever armed: a delayed poll
      * already pending is left alone, while an immediate poll (capacity just
@@ -157,6 +174,7 @@ export function createWorker(id, queue, options) {
             abortController = new AbortController();
             try {
                 state = WorkerState.RUNNING;
+                emitLifecycle("worker:started");
                 scheduleNextPoll(0);
             }
             catch (error) {
@@ -193,11 +211,17 @@ export function createWorker(id, queue, options) {
             }
             clearPollTimer();
             state = WorkerState.STOPPED;
+            emitLifecycle("worker:stopped");
         },
         async forceStop() {
+            const wasLive = state !== WorkerState.CREATED && state !== WorkerState.STOPPED;
             abortController?.abort();
             clearPollTimer();
             state = WorkerState.STOPPED;
+            // A worker that never started never stopped: reporting it would give a
+            // readiness listener a transition that did not happen.
+            if (wasLive)
+                emitLifecycle("worker:stopped");
         },
         isRunning() {
             return state === WorkerState.RUNNING;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zudojs/queue",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Background job and asynchronous task infrastructure with in-memory and adapter-based queue implementations.",
   "license": "MIT",
   "author": {
@@ -25,9 +25,9 @@
     "!dist/.tsbuildinfo"
   ],
   "dependencies": {
-    "@zudojs/errors": "1.1.0",
-    "@zudojs/constants": "1.1.0",
-    "@zudojs/serialization": "1.1.0"
+    "@zudojs/errors": "1.2.0",
+    "@zudojs/constants": "1.1.1",
+    "@zudojs/serialization": "1.1.1"
   },
   "devDependencies": {
     "typescript": "7.0.2",