bunqueue 2.8.31 → 2.8.33

package/dist/application/queueManager.js

@@ -1321,7 +1321,7 @@ export class QueueManager {
                 queue.push(parentJob);
                 shard.incrementQueued(parentId, false, parentJob.createdAt, parentJob.queue, now);
                 this.jobIndex.set(parentId, { type: 'queue', shardIdx: idx, queueName: parentJob.queue });
-                shard.notify();
+                shard.notify(parentJob.queue);
                 promoted = true;
             }
         });
@@ -1435,7 +1435,7 @@ export class QueueManager {
                 shard.getQueue(parentJob.queue).push(parentJob);
                 shard.incrementQueued(parentId, false, parentJob.createdAt, parentJob.queue, now);
                 this.jobIndex.set(parentId, { type: 'queue', shardIdx: idx, queueName: parentJob.queue });
-                shard.notify();
+                shard.notify(parentJob.queue);
                 promoted = true;
             }
         });
@@ -1529,13 +1529,17 @@ export class QueueManager {
         const ctx = this.contextFactory.getStatsContext();
         const queues = this.listQueues();
         const result = [];
+        const countsByQueue = statsMgr.getAllQueueJobCounts(queues, ctx);
         for (const name of queues) {
-            const c = statsMgr.getQueueJobCounts(name, ctx);
+            const c = countsByQueue.get(name);
+            if (!c)
+                continue;
             result.push({
                 name,
                 paused: this.isPaused(name),
                 counts: {
                     waiting: c.waiting,
+                    prioritized: c.prioritized,
                     active: c.active,
                     completed: c.completed,
                     failed: c.failed,
@@ -1545,6 +1549,15 @@ export class QueueManager {
         }
         return result;
     }
+    /** Get counts for every registered queue with one global aggregation pass. */
+    getAllQueueJobCounts() {
+        return this.getQueueJobCountsBatch(this.queueNamesCache);
+    }
+    /** Aggregate a selected group of queues without repeating global scans. */
+    getQueueJobCountsBatch(queueNames) {
+        const ctx = this.contextFactory.getStatsContext();
+        return statsMgr.getAllQueueJobCounts(queueNames, ctx);
+    }
     /** Get job counts for a specific queue */
     getQueueJobCounts(queueName) {
         return statsMgr.getQueueJobCounts(queueName, this.contextFactory.getStatsContext());

package/dist/application/queueStatsAggregator.d.ts

@@ -0,0 +1,23 @@
+import type { StatsContext } from './types';
+export interface PerQueueStats {
+    waiting: number;
+    prioritized: number;
+    delayed: number;
+    active: number;
+    dlq: number;
+}
+export interface QueueJobCounts {
+    waiting: number;
+    prioritized: number;
+    delayed: number;
+    active: number;
+    completed: number;
+    failed: number;
+    'waiting-children': number;
+    totalCompleted: number;
+    totalFailed: number;
+}
+/** Aggregate all requested queues in one pass over every global collection. */
+export declare function getAllQueueJobCounts(queueNames: Iterable<string>, ctx: StatsContext): Map<string, QueueJobCounts>;
+export declare function getPerQueueStats(ctx: StatsContext, queueNames: Set<string>): Map<string, PerQueueStats>;
+export declare function getQueueJobCounts(queueName: string, ctx: StatsContext): QueueJobCounts;

package/dist/application/queueStatsAggregator.js

@@ -0,0 +1,80 @@
+import { SHARD_COUNT, shardIndex } from '../shared/hash';
+function createCounts(queue, ctx) {
+    const metrics = ctx.perQueueMetrics?.get(queue);
+    return {
+        waiting: 0,
+        prioritized: 0,
+        delayed: 0,
+        active: 0,
+        completed: 0,
+        failed: ctx.shards[shardIndex(queue)].getDlqCount(queue),
+        'waiting-children': 0,
+        totalCompleted: Number(metrics?.totalCompleted ?? 0n),
+        totalFailed: Number(metrics?.totalFailed ?? 0n),
+    };
+}
+/** Aggregate all requested queues in one pass over every global collection. */
+export function getAllQueueJobCounts(queueNames, ctx) {
+    const result = new Map();
+    const now = Date.now();
+    for (const name of queueNames) {
+        const counts = createCounts(name, ctx);
+        const queue = ctx.shards[shardIndex(name)].queues.get(name);
+        if (queue) {
+            for (const job of queue.values()) {
+                if (job.runAt > now)
+                    counts.delayed++;
+                else if (job.priority > 0)
+                    counts.prioritized++;
+                else
+                    counts.waiting++;
+            }
+        }
+        result.set(name, counts);
+    }
+    for (const processing of ctx.processingShards) {
+        for (const job of processing.values()) {
+            const counts = result.get(job.queue);
+            if (counts)
+                counts.active++;
+        }
+    }
+    for (const [jobId, location] of ctx.jobIndex) {
+        if (location.type !== 'completed' || !ctx.completedJobs.has(jobId))
+            continue;
+        const counts = result.get(location.queueName);
+        if (counts)
+            counts.completed++;
+    }
+    for (let i = 0; i < SHARD_COUNT; i++) {
+        const shard = ctx.shards[i];
+        for (const job of shard.waitingChildren.values()) {
+            const counts = result.get(job.queue);
+            if (counts)
+                counts['waiting-children']++;
+        }
+        for (const job of shard.waitingDeps.values()) {
+            const counts = result.get(job.queue);
+            if (counts)
+                counts['waiting-children']++;
+        }
+    }
+    return result;
+}
+export function getPerQueueStats(ctx, queueNames) {
+    const allCounts = getAllQueueJobCounts(queueNames, ctx);
+    const result = new Map();
+    for (const [name, counts] of allCounts) {
+        result.set(name, {
+            waiting: counts.waiting,
+            prioritized: counts.prioritized,
+            delayed: counts.delayed,
+            active: counts.active,
+            dlq: counts.failed,
+        });
+    }
+    return result;
+}
+export function getQueueJobCounts(queueName, ctx) {
+    return getAllQueueJobCounts([queueName], ctx).get(queueName) ?? createCounts(queueName, ctx);
+}

package/dist/application/statsManager.d.ts

@@ -3,6 +3,8 @@
  * Provides system metrics and memory compaction utilities
  */
 import type { StatsContext } from './types';
+export { getAllQueueJobCounts, getPerQueueStats, getQueueJobCounts, } from './queueStatsAggregator';
+export type { PerQueueStats, QueueJobCounts } from './queueStatsAggregator';
 export interface QueueStats {
     waiting: number;
     prioritized: number;
@@ -19,13 +21,6 @@ export interface QueueStats {
     cronJobs: number;
     cronPending: number;
 }
-export interface PerQueueStats {
-    waiting: number;
-    prioritized: number;
-    delayed: number;
-    active: number;
-    dlq: number;
-}
 export interface MemoryStats {
     jobIndex: number;
     completedJobs: number;
@@ -57,25 +52,6 @@ export declare function getStats(ctx: StatsContext, cronScheduler: {
  * Returns counts of entries in all major collections.
  */
 export declare function getMemoryStats(ctx: StatsContext): MemoryStats;
-/**
- * Get per-queue statistics by iterating shards to aggregate per queue name.
- * Uses shard hashing to look up each queue in its designated shard.
- */
-export declare function getPerQueueStats(ctx: StatsContext, queueNames: Set<string>): Map<string, PerQueueStats>;
-/**
- * Get job counts for a specific queue
- */
-export declare function getQueueJobCounts(queueName: string, ctx: StatsContext): {
-    waiting: number;
-    prioritized: number;
-    delayed: number;
-    active: number;
-    completed: number;
-    failed: number;
-    'waiting-children': number;
-    totalCompleted: number;
-    totalFailed: number;
-};
 /**
  * Force compact all collections to reduce memory usage.
  * Use after large batch operations or when memory pressure is high.

package/dist/application/statsManager.js

@@ -2,17 +2,8 @@
  * Stats Manager - Queue statistics and memory management
  * Provides system metrics and memory compaction utilities
  */
-import { SHARD_COUNT, shardIndex } from '../shared/hash';
-/** Count jobs belonging to `queueName` across one or more job iterables. */
-function countByQueue(sources, queueName) {
-    let count = 0;
-    for (const src of sources) {
-        for (const job of src)
-            if (job.queue === queueName)
-                count++;
-    }
-    return count;
-}
+import { SHARD_COUNT } from '../shared/hash';
+export { getAllQueueJobCounts, getPerQueueStats, getQueueJobCounts, } from './queueStatsAggregator';
 /**
  * Get queue statistics - uses running counters + priority scan
  */
@@ -21,7 +12,6 @@ export function getStats(ctx, cronScheduler) {
     const now = Date.now();
     for (let i = 0; i < SHARD_COUNT; i++) {
         const shardStats = ctx.shards[i].getStats();
-        delayed += shardStats.delayedJobs;
         dlq += shardStats.dlqJobs;
         active += ctx.processingShards[i].size;
         // getJobState reports BOTH waitingChildren (flow parents) and waitingDeps
@@ -31,14 +21,12 @@ export function getStats(ctx, cronScheduler) {
         // Scan queues to split waiting vs prioritized (BullMQ v5 compat)
         for (const queue of ctx.shards[i].queues.values()) {
             for (const job of queue.values()) {
-                if (job.runAt <= now) {
-                    if (job.priority > 0) {
-                        prioritized++;
-                    }
-                    else {
-                        waiting++;
-                    }
-                }
+                if (job.runAt > now)
+                    delayed++;
+                else if (job.priority > 0)
+                    prioritized++;
+                else
+                    waiting++;
             }
         }
     }
@@ -103,110 +91,6 @@ export function getMemoryStats(ctx) {
         delayedHeapTotal,
     };
 }
-/**
- * Get per-queue statistics by iterating shards to aggregate per queue name.
- * Uses shard hashing to look up each queue in its designated shard.
- */
-export function getPerQueueStats(ctx, queueNames) {
-    const result = new Map();
-    const now = Date.now();
-    for (const name of queueNames) {
-        const idx = shardIndex(name);
-        const shard = ctx.shards[idx];
-        const queue = shard.queues.get(name);
-        let waiting = 0;
-        let prioritized = 0;
-        let delayed = 0;
-        if (queue) {
-            for (const job of queue.values()) {
-                if (job.runAt > now) {
-                    delayed++;
-                }
-                else if (job.priority > 0) {
-                    prioritized++;
-                }
-                else {
-                    waiting++;
-                }
-            }
-        }
-        const dlq = shard.getDlqCount(name);
-        result.set(name, { waiting, prioritized, delayed, active: 0, dlq });
-    }
-    // Count active jobs per queue from all processing shards
-    for (let i = 0; i < SHARD_COUNT; i++) {
-        for (const job of ctx.processingShards[i].values()) {
-            const entry = result.get(job.queue);
-            if (entry) {
-                entry.active++;
-            }
-        }
-    }
-    return result;
-}
-/**
- * Get job counts for a specific queue
- */
-export function getQueueJobCounts(queueName, ctx) {
-    const idx = shardIndex(queueName);
-    const shard = ctx.shards[idx];
-    const queue = shard.queues.get(queueName);
-    const now = Date.now();
-    // Count waiting vs prioritized vs delayed jobs in the queue
-    let waiting = 0;
-    let prioritized = 0;
-    let delayed = 0;
-    if (queue) {
-        for (const job of queue.values()) {
-            if (job.runAt > now) {
-                delayed++;
-            }
-            else if (job.priority > 0) {
-                prioritized++;
-            }
-            else {
-                waiting++;
-            }
-        }
-    }
-    // Count active jobs (processing) for this queue
-    let active = 0;
-    for (const procShard of ctx.processingShards) {
-        for (const job of procShard.values()) {
-            if (job.queue === queueName) {
-                active++;
-            }
-        }
-    }
-    // Count completed jobs for this queue
-    let completed = 0;
-    for (const [jobId, loc] of ctx.jobIndex) {
-        if (loc.type === 'completed' && loc.queueName === queueName && ctx.completedJobs.has(jobId)) {
-            completed++;
-        }
-    }
-    // Count failed (DLQ) jobs for this queue
-    const failed = shard.getDlq(queueName).length;
-    // Count waiting-children jobs. getJobState/getJobs treat BOTH waitingChildren
-    // (flow parents) and waitingDeps (jobs blocked on dependsOn) as 'waiting-children',
-    // so count both to stay consistent with state/list (#95 class).
-    const waitingChildrenCount = countByQueue([shard.waitingChildren.values(), shard.waitingDeps.values()], queueName);
-    // Per-queue cumulative counters
-    const perQueue = ctx.perQueueMetrics?.get(queueName);
-    const totalCompleted = Number(perQueue?.totalCompleted ?? 0n);
-    const totalFailed = Number(perQueue?.totalFailed ?? 0n);
-    return {
-        waiting,
-        prioritized,
-        delayed,
-        active,
-        completed,
-        failed,
-        'waiting-children': waitingChildrenCount,
-        totalCompleted,
-        totalFailed,
-    };
-}
 /**
  * Force compact all collections to reduce memory usage.
  * Use after large batch operations or when memory pressure is high.

package/dist/client/tcpPool.js

@@ -50,6 +50,8 @@ export class TcpConnectionPool {
                 pingInterval: this.options.pingInterval,
                 maxPingFailures: this.options.maxPingFailures,
                 maxCommandTimeouts: this.options.maxCommandTimeouts,
+                pipelining: this.options.pipelining,
+                maxInFlight: this.options.maxInFlight,
             });
             // Socket-level errors (e.g. TLS handshake failures, protocol garbage) are
             // emitted as 'error' events; without a listener EventEmitter would throw
@@ -190,7 +192,12 @@ function getPoolKey(options) {
     // TLS config must differentiate pools too: a TLS pool and a plaintext pool
     // to the same host:port are NOT interchangeable.
     const tlsKey = options?.tls ? JSON.stringify(options.tls) : '0';
-    return `${host}:${port}:${poolSize}:${tokenHash}:${tlsKey}`;
+    // pipelining/maxInFlight shape per-connection behavior: without them in the
+    // key, two Queues with different windows would silently share whichever
+    // pool was created first.
+    const pipelining = (options?.pipelining ?? true) ? '1' : '0';
+    const maxInFlight = options?.maxInFlight ?? 100;
+    return `${host}:${port}:${poolSize}:${tokenHash}:${tlsKey}:${pipelining}:${maxInFlight}`;
 }
 /** Get or create shared connection pool */
 export function getSharedPool(options) {

package/dist/domain/queue/shard.d.ts

@@ -48,9 +48,11 @@ export declare class Shard {
     /** Active FIFO groups per queue */
     readonly activeGroups: Map<string, Set<string>>;
     constructor();
-    notify(): void;
+    notify(queue?: string): void;
     notifyBatch(count: number): void;
+    notifyBatch(queue: string, count: number): void;
     waitForJob(timeoutMs: number): Promise<void>;
+    waitForJob(queue: string, timeoutMs: number): Promise<void>;
     getQueue(name: string): IndexedPriorityQueue;
     getState(name: string): QueueState;
     isPaused(name: string): boolean;

package/dist/domain/queue/shard.js

@@ -58,14 +58,22 @@ export class Shard {
         });
     }
     // ============ Waiter Management (delegated) ============
-    notify() {
-        this.waiterManager.notify();
+    notify(queue) {
+        this.waiterManager.notify(queue);
     }
-    notifyBatch(count) {
-        this.waiterManager.notifyBatch(count);
+    notifyBatch(queueOrCount, maybeCount) {
+        if (typeof queueOrCount === 'number') {
+            this.waiterManager.notifyBatch(queueOrCount);
+        }
+        else {
+            this.waiterManager.notifyBatch(queueOrCount, maybeCount ?? 0);
+        }
     }
-    waitForJob(timeoutMs) {
-        return this.waiterManager.waitForJob(timeoutMs);
+    waitForJob(queueOrTimeout, maybeTimeout) {
+        if (typeof queueOrTimeout === 'number') {
+            return this.waiterManager.waitForJob(queueOrTimeout);
+        }
+        return this.waiterManager.waitForJob(queueOrTimeout, maybeTimeout ?? 0);
     }
     // ============ Queue Operations ============
     getQueue(name) {
@@ -87,7 +95,7 @@ export class Shard {
     }
     resume(name) {
         this.limiterManager.resume(name);
-        this.waiterManager.notify();
+        this.waiterManager.notify(name);
     }
     // ============ Unique Key Management (delegated) ============
     isUniqueAvailable(queue, key) {

package/dist/domain/queue/temporalIndex.d.ts

@@ -0,0 +1,24 @@
+import type { JobId } from '../types/job';
+export interface TemporalEntry {
+    createdAt: number;
+    jobId: JobId;
+    queue: string;
+}
+/** Queue-local temporal indexes with direct lookup by job ID. */
+export declare class TemporalIndex {
+    private readonly indexesByQueue;
+    private readonly entriesByJobId;
+    private entryCount;
+    get size(): number;
+    add(createdAt: number, jobId: JobId, queue: string): void;
+    getOldJobs(queue: string, threshold: number, limit: number): Array<{
+        jobId: JobId;
+        createdAt: number;
+    }>;
+    remove(jobId: JobId): void;
+    clearQueue(queue: string): void;
+    cleanOrphaned(validJobIds: Set<JobId>): number;
+    clear(): void;
+    private removeFromQueueIndex;
+    private detachJobEntry;
+}

package/dist/domain/queue/temporalIndex.js

@@ -0,0 +1,100 @@
+import { SkipList } from '../../shared/skipList';
+const compareEntries = (a, b) => a.createdAt - b.createdAt || (a.jobId < b.jobId ? -1 : a.jobId > b.jobId ? 1 : 0);
+const createQueueIndex = () => new SkipList(compareEntries);
+/** Queue-local temporal indexes with direct lookup by job ID. */
+export class TemporalIndex {
+    indexesByQueue = new Map();
+    entriesByJobId = new Map();
+    entryCount = 0;
+    get size() {
+        return this.entryCount;
+    }
+    add(createdAt, jobId, queue) {
+        const jobEntries = this.entriesByJobId.get(jobId);
+        if (jobEntries?.some((entry) => entry.createdAt === createdAt))
+            return;
+        const entry = { createdAt, jobId, queue };
+        let queueIndex = this.indexesByQueue.get(queue);
+        if (!queueIndex) {
+            queueIndex = createQueueIndex();
+            this.indexesByQueue.set(queue, queueIndex);
+        }
+        queueIndex.insert(entry);
+        if (jobEntries)
+            jobEntries.push(entry);
+        else
+            this.entriesByJobId.set(jobId, [entry]);
+        this.entryCount++;
+    }
+    getOldJobs(queue, threshold, limit) {
+        if (limit <= 0)
+            return [];
+        const queueIndex = this.indexesByQueue.get(queue);
+        if (!queueIndex)
+            return [];
+        return queueIndex
+            .takeWhile((entry) => entry.createdAt <= threshold, limit)
+            .map(({ jobId, createdAt }) => ({ jobId, createdAt }));
+    }
+    remove(jobId) {
+        const jobEntries = this.entriesByJobId.get(jobId);
+        if (!jobEntries?.length)
+            return;
+        let targetIndex = 0;
+        for (let i = 1; i < jobEntries.length; i++) {
+            if (compareEntries(jobEntries[i], jobEntries[targetIndex]) < 0)
+                targetIndex = i;
+        }
+        const [entry] = jobEntries.splice(targetIndex, 1);
+        this.removeFromQueueIndex(entry);
+        if (jobEntries.length === 0)
+            this.entriesByJobId.delete(jobId);
+        this.entryCount--;
+    }
+    clearQueue(queue) {
+        const queueIndex = this.indexesByQueue.get(queue);
+        if (!queueIndex)
+            return;
+        for (const entry of queueIndex.values())
+            this.detachJobEntry(entry);
+        this.entryCount -= queueIndex.size;
+        this.indexesByQueue.delete(queue);
+    }
+    cleanOrphaned(validJobIds) {
+        let removed = 0;
+        for (const [jobId, entries] of this.entriesByJobId) {
+            if (validJobIds.has(jobId))
+                continue;
+            for (const entry of entries) {
+                this.removeFromQueueIndex(entry);
+                removed++;
+            }
+            this.entriesByJobId.delete(jobId);
+        }
+        this.entryCount -= removed;
+        return removed;
+    }
+    clear() {
+        this.indexesByQueue.clear();
+        this.entriesByJobId.clear();
+        this.entryCount = 0;
+    }
+    removeFromQueueIndex(entry) {
+        const queueIndex = this.indexesByQueue.get(entry.queue);
+        if (!queueIndex)
+            return;
+        queueIndex.delete(entry);
+        if (queueIndex.isEmpty)
+            this.indexesByQueue.delete(entry.queue);
+    }
+    detachJobEntry(entry) {
+        const jobEntries = this.entriesByJobId.get(entry.jobId);
+        if (!jobEntries)
+            return;
+        const index = jobEntries.indexOf(entry);
+        if (index !== -1)
+            jobEntries.splice(index, 1);
+        if (jobEntries.length === 0)
+            this.entriesByJobId.delete(entry.jobId);
+    }
+}

package/dist/domain/queue/temporalManager.d.ts

@@ -6,22 +6,12 @@
  * - Delayed heap (MinHeap) for O(k) delayed job refresh
  */
 import type { JobId } from '../types/job';
-/** Entry in the temporal index */
-export interface TemporalEntry {
-    createdAt: number;
-    jobId: JobId;
-    queue: string;
-}
+export type { TemporalEntry } from './temporalIndex';
 /**
  * Manages temporal indexing for efficient job cleanup
  * and delayed job tracking for O(k) refresh
  */
 export declare class TemporalManager {
-    /**
-     * Temporal index: Skip List for O(log n) insert/delete
-     * Ordered by createdAt for efficient cleanQueue range queries
-     * Uses equality check on jobId to prevent duplicate entries for the same job
-     */
     private readonly temporalIndex;
     /** Set of delayed job IDs for tracking when they become ready */
     private readonly delayedJobIds;
@@ -65,6 +55,7 @@ export declare class TemporalManager {
      * Returns number of jobs that became ready
      */
     refreshDelayed(now: number): number;
+    private maybeCompactDelayedHeap;
     /** Get delayed job count */
     get delayedCount(): number;
     /** Clear all delayed tracking */