@zudojs/queue 1.2.0 → 1.4.0

package/README.md

@@ -69,8 +69,72 @@ await queue.close();
 - Draining shutdown with a bounded `closeTimeout`, then cooperative abort
 - Stalled-job reclaim (`stalledAfter`, `maxStalledCount`) for consumers that die
   mid-job
-- Bounded retention of settled jobs, so a long-lived queue does not grow without
-  limit
+- Bounded retention of settled jobs and of dead-lettered jobs, so a long-lived
+  queue does not grow without limit
+
+## Polling and process lifetime
+
+The queue's own consumer polls every `pollInterval` milliseconds when you set
+one; unset, it starts at 50 ms and backs off to 2000 ms while idle. It never
+waits out that interval when work arrives: `add()`, a delayed job coming due, a
+retry's backoff elapsing, `resume()` and a finished job all wake it at once. A
+`Worker` is woken the same way through `queue.onJobReady`, so its
+`pollInterval` only bounds how often an idle worker re-checks.
+
+Pending work keeps the Node.js process alive. While the queue has waiting,
+delayed, retrying or running jobs that one of its processors can run, a script
+whose only work is the queue does not exit before the jobs run. A started
+`Worker` keeps the process alive until `stop()` or `forceStop()`. An idle queue,
+a paused one, work that no processor handles, and a closed queue never hold the
+process open. Pass `keepAlive: false` (to the queue or the worker) for the old
+behaviour, where every timer was unreferenced.
+
+## Payloads
+
+Payloads round-trip through the serializer on `add()`, so the stored job is a
+copy. The default `JsonSerializer` preserves `Date`, `BigInt`, `Map`, `Set`,
+`Uint8Array` and `Error`, so a field typed `Date` arrives as a `Date`:
+
+```typescript
+const reminders = createInMemoryQueue<{ at: Date }>(createQueueName("reminders"));
+reminders.process("remind", async (job) => job.data.at.getTime()); // a real Date
+```
+
+`createJsonSerializer({ preserveTypes: false })` gives plain JSON, where a
+`Date` becomes its ISO string (type it as `string`). `PassthroughSerializer`
+(or `serializePayloads: false`) stores payloads by reference, class instances
+included, with no copy and no serialization.
+
+## Ordering
+
+Higher `priority` runs first. Within a priority, jobs run in the order they
+became runnable: when they were added, or for a delayed job when its delay
+elapsed, so a delayed job never jumps ahead of jobs that were already waiting
+when it came due. A retried job keeps its original place in line.
+
+## Attempts
+
+`job.attempt` counts the attempts already made, so it is `0` while the first
+attempt runs. `context.attemptNumber` is the 1-based number of the attempt in
+progress (`job.attempt + 1`), the same convention as `ctx.attempt` in
+`@zudojs/scheduler`:
+
+```typescript
+queue.process("sync", async (job, context) => {
+  context.log(`attempt ${context.attemptNumber} of ${job.maxAttempts}`);
+});
+```
+
+## Events
+
+`queue.events` works without configuration: a queue created without an
+`eventEmitter` gets an in-memory one.
+
+```typescript
+queue.events?.on("job:completed", ({ job, result }) => {
+  console.log(job.id, result);
+});
+```
 
 ## Workers, timeouts and context
 
@@ -120,6 +184,10 @@ Keep captured values small and serializable (ids, not live objects). With
 `@zudojs/tenancy`, capture the tenant id and restore it with the tenant context
 storage's `run`.
 
+The `zudo:context` metadata key is reserved: it is written only by the queue's
+own carriers and is stripped from any `metadata` passed to `add()`, so an
+enqueuer cannot choose the context its job runs under.
+
 Errors with no caller to receive them (a worker's failing poll, a throwing event
 listener) go to `logger.error` when a logger is configured, and otherwise to
 `process.emitWarning`, never to `console`.

package/dist/contextCarrier/contextCarrier.core.d.ts

@@ -13,10 +13,15 @@ export declare const CONTEXT_METADATA_KEY = "zudo:context";
 /**
  * Captures every carrier's value into a metadata record.
  *
+ * {@link CONTEXT_METADATA_KEY} is owned by the queue: a value the caller put
+ * under that key is always discarded, whether or not a carrier captured
+ * anything, so an enqueuer cannot forge the tenant, correlation id or trace
+ * the processor runs under.
+ *
  * @param carriers - The queue's context carriers.
  * @param metadata - The job's own metadata, if any.
  * @returns The metadata with a {@link CONTEXT_METADATA_KEY} entry added, or
- *   the input unchanged when no carrier captured anything.
+ *   without one when no carrier captured anything.
  */
 export declare function captureContext(carriers: readonly QueueContextCarrier[] | undefined, metadata: Record<string, unknown> | undefined): Record<string, unknown> | undefined;
 /**

package/dist/contextCarrier/contextCarrier.core.js

@@ -8,17 +8,37 @@
  * `{ [carrier.key]: value }` record.
  */
 export const CONTEXT_METADATA_KEY = "zudo:context";
+/**
+ * Strips the reserved context key from caller-supplied metadata.
+ *
+ * The key is owned by the queue on every path: whatever the enqueuer put
+ * there is dropped, so `add()` metadata can never supply a context record of
+ * its own. Returns the input untouched when there is nothing to strip, so the
+ * common case still allocates nothing.
+ */
+function withoutStoredContext(metadata) {
+    if (!metadata || !Object.hasOwn(metadata, CONTEXT_METADATA_KEY)) {
+        return metadata;
+    }
+    const { [CONTEXT_METADATA_KEY]: _discarded, ...rest } = metadata;
+    return rest;
+}
 /**
  * Captures every carrier's value into a metadata record.
  *
+ * {@link CONTEXT_METADATA_KEY} is owned by the queue: a value the caller put
+ * under that key is always discarded, whether or not a carrier captured
+ * anything, so an enqueuer cannot forge the tenant, correlation id or trace
+ * the processor runs under.
+ *
  * @param carriers - The queue's context carriers.
  * @param metadata - The job's own metadata, if any.
  * @returns The metadata with a {@link CONTEXT_METADATA_KEY} entry added, or
- *   the input unchanged when no carrier captured anything.
+ *   without one when no carrier captured anything.
  */
 export function captureContext(carriers, metadata) {
     if (!carriers || carriers.length === 0)
-        return metadata;
+        return withoutStoredContext(metadata);
     const captured = {};
     let any = false;
     for (const carrier of carriers) {
@@ -29,7 +49,7 @@ export function captureContext(carriers, metadata) {
         any = true;
     }
     if (!any)
-        return metadata;
+        return withoutStoredContext(metadata);
     return { ...metadata, [CONTEXT_METADATA_KEY]: Object.freeze(captured) };
 }
 /**

package/dist/deadLetter/deadLetter.core.d.ts

@@ -1,9 +1,34 @@
 import type { Job } from "../job/job.type.js";
 import type { DeadLetterStore } from "./deadLetter.type.js";
+/**
+ * Dead-lettered jobs retained by the in-memory store when no cap is given.
+ *
+ * Mirrors the queue's own `DEFAULT_RETAINED_JOBS`: the dead letter store
+ * keeps a full copy of every failed job — payload, metadata and error — so an
+ * unbounded one grows for the life of the process at whatever the failure
+ * rate happens to be.
+ */
+export declare const DEFAULT_DEAD_LETTER_JOBS = 1000;
+/**
+ * Options for {@link createInMemoryDeadLetterStore}.
+ */
+export interface InMemoryDeadLetterStoreOptions {
+    /**
+     * Maximum number of dead-lettered jobs retained before the oldest are
+     * evicted. Defaults to {@link DEFAULT_DEAD_LETTER_JOBS}. Pass
+     * `Number.POSITIVE_INFINITY` for the previous unbounded behaviour.
+     */
+    readonly maxEntries?: number;
+}
 /**
  * Creates an in-memory dead letter store.
+ *
+ * Retention is bounded: past `maxEntries` the oldest entry is evicted, so a
+ * long-lived queue with a steady failure rate does not grow without limit.
+ *
+ * @param options - Retention options.
  */
-export declare function createInMemoryDeadLetterStore<TData = unknown>(): DeadLetterStore<TData>;
+export declare function createInMemoryDeadLetterStore<TData = unknown>(options?: InMemoryDeadLetterStoreOptions): DeadLetterStore<TData>;
 /**
  * Moves a failed job to the dead letter store.
  */

package/dist/deadLetter/deadLetter.core.js

@@ -1,12 +1,38 @@
 import { toQueueError } from "@zudojs/errors";
+/**
+ * Dead-lettered jobs retained by the in-memory store when no cap is given.
+ *
+ * Mirrors the queue's own `DEFAULT_RETAINED_JOBS`: the dead letter store
+ * keeps a full copy of every failed job — payload, metadata and error — so an
+ * unbounded one grows for the life of the process at whatever the failure
+ * rate happens to be.
+ */
+export const DEFAULT_DEAD_LETTER_JOBS = 1_000;
 /**
  * Creates an in-memory dead letter store.
+ *
+ * Retention is bounded: past `maxEntries` the oldest entry is evicted, so a
+ * long-lived queue with a steady failure rate does not grow without limit.
+ *
+ * @param options - Retention options.
  */
-export function createInMemoryDeadLetterStore() {
+export function createInMemoryDeadLetterStore(options) {
     const store = new Map();
+    const configured = options?.maxEntries ?? DEFAULT_DEAD_LETTER_JOBS;
+    const maxEntries = configured > 0 ? configured : DEFAULT_DEAD_LETTER_JOBS;
     return {
         async add(deadLetterJob) {
+            // Re-inserting an existing id must not count as a new entry, so
+            // delete first: a Map keeps its original insertion order otherwise,
+            // which would also evict the wrong entry.
+            store.delete(deadLetterJob.job.id);
             store.set(deadLetterJob.job.id, deadLetterJob);
+            while (store.size > maxEntries) {
+                const oldest = store.keys().next().value;
+                if (oldest === undefined)
+                    break;
+                store.delete(oldest);
+            }
         },
         async get(jobId) {
             return store.get(jobId) ?? null;

package/dist/deadLetter/index.d.ts

@@ -4,6 +4,7 @@
  * Provides storage and management for jobs that have
  * exceeded their retry attempts.
  */
-export { createInMemoryDeadLetterStore, moveToDeadLetter, } from "./deadLetter.core.js";
+export { DEFAULT_DEAD_LETTER_JOBS, createInMemoryDeadLetterStore, moveToDeadLetter, } from "./deadLetter.core.js";
+export type { InMemoryDeadLetterStoreOptions } from "./deadLetter.core.js";
 export type { DeadLetterJob, DeadLetterStore } from "./deadLetter.type.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/deadLetter/index.js

@@ -4,5 +4,5 @@
  * Provides storage and management for jobs that have
  * exceeded their retry attempts.
  */
-export { createInMemoryDeadLetterStore, moveToDeadLetter, } from "./deadLetter.core.js";
+export { DEFAULT_DEAD_LETTER_JOBS, createInMemoryDeadLetterStore, moveToDeadLetter, } from "./deadLetter.core.js";
 //# sourceMappingURL=index.js.map

package/dist/inMemoryQueue/inMemoryQueue.core.d.ts

@@ -4,6 +4,7 @@ import type { Queue, QueueOptions, QueueStats } from "../queue/queue.type.js";
 import type { Processor } from "../processor/processor.type.js";
 import type { JobOptions } from "../jobOptions/jobOptions.type.js";
 import type { QueueMiddleware } from "../middleware/middleware.type.js";
+import type { QueueEventEmitter } from "../queueEmitter/queueEmitter.type.js";
 import type { DeadLetterJob } from "../deadLetter/deadLetter.type.js";
 /**
  * In-memory queue implementation.
@@ -23,7 +24,9 @@ export declare class InMemoryQueue<TData = unknown> implements Queue<TData> {
     /** Whether the internal poller claims jobs; see `setAutoProcess`. */
     private autoProcess;
     private activeCount;
-    private pollTimer;
+    private readonly poller;
+    /** Consumers to tell when a job may have become runnable. */
+    private readonly readyListeners;
     private readonly scheduledTimers;
     private readonly retryTimers;
     private readonly inFlight;
@@ -32,11 +35,23 @@ export declare class InMemoryQueue<TData = unknown> implements Queue<TData> {
     private readonly stalledCounts;
     private readonly deduplicationIndex;
     private readonly deadLetterStore;
+    /**
+     * Whether this queue created its own dead letter store. A store handed in
+     * by the caller outlives the queue and is theirs to clear.
+     */
+    private readonly ownsDeadLetterStore;
     private readonly emitter;
     private readonly counters;
-    private emptySince;
-    private backoffMs;
     constructor(name: QueueName, options?: QueueOptions);
+    /**
+     * Subscribes to "a job may have become runnable": one was added, released
+     * or reclaimed, a delay or retry backoff elapsed, or the queue resumed.
+     *
+     * A `Worker` uses this to claim immediately instead of on its next poll.
+     *
+     * @returns A function that unsubscribes.
+     */
+    onJobReady(listener: () => void): () => void;
     /**
      * Turns the internal poller on or off as a consumer.
      *
@@ -46,6 +61,14 @@ export declare class InMemoryQueue<TData = unknown> implements Queue<TData> {
      * concurrency apply to every job.
      */
     setAutoProcess(enabled: boolean): void;
+    /**
+     * The emitter this queue publishes lifecycle events on.
+     *
+     * An in-memory emitter when the queue was created without one, so
+     * `queue.events.on(...)` works out of the box and a worker can report its
+     * lifecycle unconditionally.
+     */
+    get events(): QueueEventEmitter;
     add(jobName: string, data: TData, options?: JobOptions): Promise<Job<TData>>;
     process(name: string, processor: Processor<TData>): void;
     getJob(jobId: JobId): Promise<Job<TData> | null>;
@@ -100,17 +123,18 @@ export declare class InMemoryQueue<TData = unknown> implements Queue<TData> {
      */
     private encodePayload;
     /**
-     * Selects the highest-priority job that is due and runnable.
-     *
-     * Ties on priority are broken by creation time, oldest first. The
-     * incumbent is tracked by reference rather than by a sentinel priority,
-     * so jobs with negative priorities are selectable like any other.
+     * Selects the next job that is due and runnable; see {@link selectNextJob}
+     * for the ordering.
      */
     private selectJob;
-    private startPolling;
-    private scheduleNextTick;
-    private scheduleTick;
-    private stopPolling;
+    /**
+     * Whether the poll loop should hold the process open: while the queue is
+     * consuming and has work one of its processors can run. Work nothing can
+     * consume, a paused queue and `keepAlive: false` never do.
+     */
+    private shouldKeepAlive;
+    /** Wakes the poll loop and tells consumers a job may be runnable. */
+    private jobReady;
     /**
      * Runs an already-claimed job through this queue's processing pipeline.
      *
@@ -125,6 +149,12 @@ export declare class InMemoryQueue<TData = unknown> implements Queue<TData> {
         /** Fallback timeout for a job that carries none of its own. */
         timeoutMs?: number;
     }): Promise<void>;
+    /**
+     * One poll: promotes due delayed jobs, claims runnable jobs up to the
+     * concurrency limit, and reclaims stalled ones.
+     *
+     * @returns Whether any job was dispatched.
+     */
     private processTick;
     /**
      * Returns jobs stuck in `active` to the waiting pool.