bunqueue 2.8.32 → 2.8.34

package/dist/application/statsManager.js

@@ -2,17 +2,8 @@
  * Stats Manager - Queue statistics and memory management
  * Provides system metrics and memory compaction utilities
  */
-import { SHARD_COUNT, shardIndex } from '../shared/hash';
-/** Count jobs belonging to `queueName` across one or more job iterables. */
-function countByQueue(sources, queueName) {
-    let count = 0;
-    for (const src of sources) {
-        for (const job of src)
-            if (job.queue === queueName)
-                count++;
-    }
-    return count;
-}
+import { SHARD_COUNT } from '../shared/hash';
+export { getAllQueueJobCounts, getPerQueueStats, getQueueJobCounts, } from './queueStatsAggregator';
 /**
  * Get queue statistics - uses running counters + priority scan
  */
@@ -21,7 +12,6 @@ export function getStats(ctx, cronScheduler) {
     const now = Date.now();
     for (let i = 0; i < SHARD_COUNT; i++) {
         const shardStats = ctx.shards[i].getStats();
-        delayed += shardStats.delayedJobs;
         dlq += shardStats.dlqJobs;
         active += ctx.processingShards[i].size;
         // getJobState reports BOTH waitingChildren (flow parents) and waitingDeps
@@ -31,14 +21,12 @@ export function getStats(ctx, cronScheduler) {
         // Scan queues to split waiting vs prioritized (BullMQ v5 compat)
         for (const queue of ctx.shards[i].queues.values()) {
             for (const job of queue.values()) {
-                if (job.runAt <= now) {
-                    if (job.priority > 0) {
-                        prioritized++;
-                    }
-                    else {
-                        waiting++;
-                    }
-                }
+                if (job.runAt > now)
+                    delayed++;
+                else if (job.priority > 0)
+                    prioritized++;
+                else
+                    waiting++;
             }
         }
     }
@@ -103,110 +91,6 @@ export function getMemoryStats(ctx) {
         delayedHeapTotal,
     };
 }
-/**
- * Get per-queue statistics by iterating shards to aggregate per queue name.
- * Uses shard hashing to look up each queue in its designated shard.
- */
-export function getPerQueueStats(ctx, queueNames) {
-    const result = new Map();
-    const now = Date.now();
-    for (const name of queueNames) {
-        const idx = shardIndex(name);
-        const shard = ctx.shards[idx];
-        const queue = shard.queues.get(name);
-        let waiting = 0;
-        let prioritized = 0;
-        let delayed = 0;
-        if (queue) {
-            for (const job of queue.values()) {
-                if (job.runAt > now) {
-                    delayed++;
-                }
-                else if (job.priority > 0) {
-                    prioritized++;
-                }
-                else {
-                    waiting++;
-                }
-            }
-        }
-        const dlq = shard.getDlqCount(name);
-        result.set(name, { waiting, prioritized, delayed, active: 0, dlq });
-    }
-    // Count active jobs per queue from all processing shards
-    for (let i = 0; i < SHARD_COUNT; i++) {
-        for (const job of ctx.processingShards[i].values()) {
-            const entry = result.get(job.queue);
-            if (entry) {
-                entry.active++;
-            }
-        }
-    }
-    return result;
-}
-/**
- * Get job counts for a specific queue
- */
-export function getQueueJobCounts(queueName, ctx) {
-    const idx = shardIndex(queueName);
-    const shard = ctx.shards[idx];
-    const queue = shard.queues.get(queueName);
-    const now = Date.now();
-    // Count waiting vs prioritized vs delayed jobs in the queue
-    let waiting = 0;
-    let prioritized = 0;
-    let delayed = 0;
-    if (queue) {
-        for (const job of queue.values()) {
-            if (job.runAt > now) {
-                delayed++;
-            }
-            else if (job.priority > 0) {
-                prioritized++;
-            }
-            else {
-                waiting++;
-            }
-        }
-    }
-    // Count active jobs (processing) for this queue
-    let active = 0;
-    for (const procShard of ctx.processingShards) {
-        for (const job of procShard.values()) {
-            if (job.queue === queueName) {
-                active++;
-            }
-        }
-    }
-    // Count completed jobs for this queue
-    let completed = 0;
-    for (const [jobId, loc] of ctx.jobIndex) {
-        if (loc.type === 'completed' && loc.queueName === queueName && ctx.completedJobs.has(jobId)) {
-            completed++;
-        }
-    }
-    // Count failed (DLQ) jobs for this queue
-    const failed = shard.getDlq(queueName).length;
-    // Count waiting-children jobs. getJobState/getJobs treat BOTH waitingChildren
-    // (flow parents) and waitingDeps (jobs blocked on dependsOn) as 'waiting-children',
-    // so count both to stay consistent with state/list (#95 class).
-    const waitingChildrenCount = countByQueue([shard.waitingChildren.values(), shard.waitingDeps.values()], queueName);
-    // Per-queue cumulative counters
-    const perQueue = ctx.perQueueMetrics?.get(queueName);
-    const totalCompleted = Number(perQueue?.totalCompleted ?? 0n);
-    const totalFailed = Number(perQueue?.totalFailed ?? 0n);
-    return {
-        waiting,
-        prioritized,
-        delayed,
-        active,
-        completed,
-        failed,
-        'waiting-children': waitingChildrenCount,
-        totalCompleted,
-        totalFailed,
-    };
-}
 /**
  * Force compact all collections to reduce memory usage.
  * Use after large batch operations or when memory pressure is high.

package/dist/client/queue/operations/management.js

@@ -93,17 +93,7 @@ export async function cleanAsync(ctx, grace, limit, type) {
 /** Promote delayed jobs to waiting */
 export async function promoteJobs(ctx, opts) {
     if (ctx.embedded) {
-        // Get delayed jobs and promote them one by one
-        const manager = getSharedManager();
-        const jobs = manager.getJobs(ctx.name, { state: 'delayed' });
-        const count = opts?.count ?? jobs.length;
-        let promoted = 0;
-        for (let i = 0; i < Math.min(count, jobs.length); i++) {
-            const success = await manager.promote(jobs[i].id);
-            if (success)
-                promoted++;
-        }
-        return promoted;
+        return getSharedManager().promoteJobs(ctx.name, opts?.count);
     }
     const response = await ctx.tcp.send({
         cmd: 'PromoteJobs',

package/dist/domain/queue/shard.d.ts

@@ -48,9 +48,11 @@ export declare class Shard {
     /** Active FIFO groups per queue */
     readonly activeGroups: Map<string, Set<string>>;
     constructor();
-    notify(): void;
+    notify(queue?: string): void;
     notifyBatch(count: number): void;
+    notifyBatch(queue: string, count: number): void;
     waitForJob(timeoutMs: number): Promise<void>;
+    waitForJob(queue: string, timeoutMs: number): Promise<void>;
     getQueue(name: string): IndexedPriorityQueue;
     getState(name: string): QueueState;
     isPaused(name: string): boolean;
@@ -114,6 +116,8 @@ export declare class Shard {
     };
     incrementQueued(jobId: JobId, isDelayed: boolean, createdAt?: number, queue?: string, runAt?: number): void;
     decrementQueued(jobId: JobId): void;
+    /** Remove a promoted job from delayed tracking without changing queue depth. */
+    promoteDelayed(jobId: JobId): void;
     incrementDlq(): void;
     decrementDlq(count?: number): void;
     refreshDelayedCount(now: number): void;

package/dist/domain/queue/shard.js

@@ -58,14 +58,22 @@ export class Shard {
         });
     }
     // ============ Waiter Management (delegated) ============
-    notify() {
-        this.waiterManager.notify();
+    notify(queue) {
+        this.waiterManager.notify(queue);
     }
-    notifyBatch(count) {
-        this.waiterManager.notifyBatch(count);
+    notifyBatch(queueOrCount, maybeCount) {
+        if (typeof queueOrCount === 'number') {
+            this.waiterManager.notifyBatch(queueOrCount);
+        }
+        else {
+            this.waiterManager.notifyBatch(queueOrCount, maybeCount ?? 0);
+        }
     }
-    waitForJob(timeoutMs) {
-        return this.waiterManager.waitForJob(timeoutMs);
+    waitForJob(queueOrTimeout, maybeTimeout) {
+        if (typeof queueOrTimeout === 'number') {
+            return this.waiterManager.waitForJob(queueOrTimeout);
+        }
+        return this.waiterManager.waitForJob(queueOrTimeout, maybeTimeout ?? 0);
     }
     // ============ Queue Operations ============
     getQueue(name) {
@@ -87,7 +95,7 @@ export class Shard {
     }
     resume(name) {
         this.limiterManager.resume(name);
-        this.waiterManager.notify();
+        this.waiterManager.notify(name);
     }
     // ============ Unique Key Management (delegated) ============
     isUniqueAvailable(queue, key) {
@@ -294,6 +302,11 @@ export class Shard {
     decrementQueued(jobId) {
         this.counters.decrementQueued(jobId);
     }
+    /** Remove a promoted job from delayed tracking without changing queue depth. */
+    promoteDelayed(jobId) {
+        this.temporalManager.removeDelayed(jobId);
+        this.counters.syncDelayedCount();
+    }
     incrementDlq() {
         this.counters.incrementDlq();
     }

package/dist/domain/queue/temporalIndex.d.ts

@@ -0,0 +1,24 @@
+import type { JobId } from '../types/job';
+export interface TemporalEntry {
+    createdAt: number;
+    jobId: JobId;
+    queue: string;
+}
+/** Queue-local temporal indexes with direct lookup by job ID. */
+export declare class TemporalIndex {
+    private readonly indexesByQueue;
+    private readonly entriesByJobId;
+    private entryCount;
+    get size(): number;
+    add(createdAt: number, jobId: JobId, queue: string): void;
+    getOldJobs(queue: string, threshold: number, limit: number): Array<{
+        jobId: JobId;
+        createdAt: number;
+    }>;
+    remove(jobId: JobId): void;
+    clearQueue(queue: string): void;
+    cleanOrphaned(validJobIds: Set<JobId>): number;
+    clear(): void;
+    private removeFromQueueIndex;
+    private detachJobEntry;
+}

package/dist/domain/queue/temporalIndex.js

@@ -0,0 +1,100 @@
+import { SkipList } from '../../shared/skipList';
+const compareEntries = (a, b) => a.createdAt - b.createdAt || (a.jobId < b.jobId ? -1 : a.jobId > b.jobId ? 1 : 0);
+const createQueueIndex = () => new SkipList(compareEntries);
+/** Queue-local temporal indexes with direct lookup by job ID. */
+export class TemporalIndex {
+    indexesByQueue = new Map();
+    entriesByJobId = new Map();
+    entryCount = 0;
+    get size() {
+        return this.entryCount;
+    }
+    add(createdAt, jobId, queue) {
+        const jobEntries = this.entriesByJobId.get(jobId);
+        if (jobEntries?.some((entry) => entry.createdAt === createdAt))
+            return;
+        const entry = { createdAt, jobId, queue };
+        let queueIndex = this.indexesByQueue.get(queue);
+        if (!queueIndex) {
+            queueIndex = createQueueIndex();
+            this.indexesByQueue.set(queue, queueIndex);
+        }
+        queueIndex.insert(entry);
+        if (jobEntries)
+            jobEntries.push(entry);
+        else
+            this.entriesByJobId.set(jobId, [entry]);
+        this.entryCount++;
+    }
+    getOldJobs(queue, threshold, limit) {
+        if (limit <= 0)
+            return [];
+        const queueIndex = this.indexesByQueue.get(queue);
+        if (!queueIndex)
+            return [];
+        return queueIndex
+            .takeWhile((entry) => entry.createdAt <= threshold, limit)
+            .map(({ jobId, createdAt }) => ({ jobId, createdAt }));
+    }
+    remove(jobId) {
+        const jobEntries = this.entriesByJobId.get(jobId);
+        if (!jobEntries?.length)
+            return;
+        let targetIndex = 0;
+        for (let i = 1; i < jobEntries.length; i++) {
+            if (compareEntries(jobEntries[i], jobEntries[targetIndex]) < 0)
+                targetIndex = i;
+        }
+        const [entry] = jobEntries.splice(targetIndex, 1);
+        this.removeFromQueueIndex(entry);
+        if (jobEntries.length === 0)
+            this.entriesByJobId.delete(jobId);
+        this.entryCount--;
+    }
+    clearQueue(queue) {
+        const queueIndex = this.indexesByQueue.get(queue);
+        if (!queueIndex)
+            return;
+        for (const entry of queueIndex.values())
+            this.detachJobEntry(entry);
+        this.entryCount -= queueIndex.size;
+        this.indexesByQueue.delete(queue);
+    }
+    cleanOrphaned(validJobIds) {
+        let removed = 0;
+        for (const [jobId, entries] of this.entriesByJobId) {
+            if (validJobIds.has(jobId))
+                continue;
+            for (const entry of entries) {
+                this.removeFromQueueIndex(entry);
+                removed++;
+            }
+            this.entriesByJobId.delete(jobId);
+        }
+        this.entryCount -= removed;
+        return removed;
+    }
+    clear() {
+        this.indexesByQueue.clear();
+        this.entriesByJobId.clear();
+        this.entryCount = 0;
+    }
+    removeFromQueueIndex(entry) {
+        const queueIndex = this.indexesByQueue.get(entry.queue);
+        if (!queueIndex)
+            return;
+        queueIndex.delete(entry);
+        if (queueIndex.isEmpty)
+            this.indexesByQueue.delete(entry.queue);
+    }
+    detachJobEntry(entry) {
+        const jobEntries = this.entriesByJobId.get(entry.jobId);
+        if (!jobEntries)
+            return;
+        const index = jobEntries.indexOf(entry);
+        if (index !== -1)
+            jobEntries.splice(index, 1);
+        if (jobEntries.length === 0)
+            this.entriesByJobId.delete(entry.jobId);
+    }
+}

package/dist/domain/queue/temporalManager.d.ts

@@ -6,22 +6,12 @@
  * - Delayed heap (MinHeap) for O(k) delayed job refresh
  */
 import type { JobId } from '../types/job';
-/** Entry in the temporal index */
-export interface TemporalEntry {
-    createdAt: number;
-    jobId: JobId;
-    queue: string;
-}
+export type { TemporalEntry } from './temporalIndex';
 /**
  * Manages temporal indexing for efficient job cleanup
  * and delayed job tracking for O(k) refresh
  */
 export declare class TemporalManager {
-    /**
-     * Temporal index: Skip List for O(log n) insert/delete
-     * Ordered by createdAt for efficient cleanQueue range queries
-     * Uses equality check on jobId to prevent duplicate entries for the same job
-     */
     private readonly temporalIndex;
     /** Set of delayed job IDs for tracking when they become ready */
     private readonly delayedJobIds;
@@ -65,6 +55,7 @@ export declare class TemporalManager {
      * Returns number of jobs that became ready
      */
     refreshDelayed(now: number): number;
+    private maybeCompactDelayedHeap;
     /** Get delayed job count */
     get delayedCount(): number;
     /** Clear all delayed tracking */

package/dist/domain/queue/temporalManager.js

@@ -5,29 +5,15 @@
  * - Temporal index (SkipList) for efficient cleanQueue range queries
  * - Delayed heap (MinHeap) for O(k) delayed job refresh
  */
-import { SkipList } from '../../shared/skipList';
 import { MinHeap } from '../../shared/minHeap';
+import { TemporalIndex } from './temporalIndex';
+const DELAYED_COMPACTION_MIN_STALE = 256;
 /**
  * Manages temporal indexing for efficient job cleanup
  * and delayed job tracking for O(k) refresh
  */
 export class TemporalManager {
-    /**
-     * Temporal index: Skip List for O(log n) insert/delete
-     * Ordered by createdAt for efficient cleanQueue range queries
-     * Uses equality check on jobId to prevent duplicate entries for the same job
-     */
-    // Total-order comparator: createdAt first, then jobId as a tie-break. Without
-    // the tie-break, a batch of jobs sharing one createdAt (the addBulk case —
-    // `now` is captured once) makes every node compare-equal, which (a) turns
-    // SkipList.insert's duplicate-check scan into O(n) per insert => O(n²) for the
-    // batch, and (b) makes SkipList.delete remove the WRONG same-createdAt node
-    // (it stops at the first compare-equal node). A total order fixes both: the
-    // dedup scan and delete both resolve to the exact (createdAt, jobId) node in
-    // O(log n). jobId is a string (UUIDv7 by default, or any custom id), so its
-    // lexicographic comparison is a valid total order in every case. Equality is
-    // still by jobId (now reached only for a true duplicate).
-    temporalIndex = new SkipList((a, b) => a.createdAt - b.createdAt || (a.jobId < b.jobId ? -1 : a.jobId > b.jobId ? 1 : 0), 16, 0.5, (a, b) => a.jobId === b.jobId);
+    temporalIndex = new TemporalIndex();
     /** Set of delayed job IDs for tracking when they become ready */
     delayedJobIds = new Set();
     /**
@@ -40,7 +26,7 @@ export class TemporalManager {
     // ============ Temporal Index Operations ============
     /** Add job to temporal index - O(log n) */
     addToIndex(createdAt, jobId, queue) {
-        this.temporalIndex.insert({ createdAt, jobId, queue });
+        this.temporalIndex.add(createdAt, jobId, queue);
     }
     /**
      * Get old jobs from temporal index - O(log n + k)
@@ -49,36 +35,22 @@ export class TemporalManager {
     getOldJobs(queue, thresholdMs, limit) {
         const now = Date.now();
         const threshold = now - thresholdMs;
-        const result = [];
-        for (const entry of this.temporalIndex.values()) {
-            if (entry.createdAt > threshold)
-                break;
-            if (entry.queue === queue) {
-                result.push({ jobId: entry.jobId, createdAt: entry.createdAt });
-                if (result.length >= limit)
-                    break;
-            }
-        }
-        return result;
+        return this.temporalIndex.getOldJobs(queue, threshold, limit);
     }
     /** Remove job from temporal index */
     removeFromIndex(jobId) {
-        this.temporalIndex.deleteWhere((e) => e.jobId === jobId);
+        this.temporalIndex.remove(jobId);
     }
     /** Clear temporal index for a queue */
     clearIndexForQueue(queue) {
-        this.temporalIndex.removeAll((e) => e.queue === queue);
+        this.temporalIndex.clearQueue(queue);
     }
     /**
      * Clean orphaned temporal index entries.
      * Removes entries for jobs that no longer exist.
      */
     cleanOrphaned(validJobIds) {
-        if (this.temporalIndex.size === 0)
-            return 0;
-        const beforeSize = this.temporalIndex.size;
-        this.temporalIndex.removeAll((e) => !validJobIds.has(e.jobId));
-        return beforeSize - this.temporalIndex.size;
+        return this.temporalIndex.cleanOrphaned(validJobIds);
     }
     /** Get temporal index size */
     get indexSize() {
@@ -94,15 +66,15 @@ export class TemporalManager {
         this.delayedJobIds.add(jobId);
         this.delayedHeap.push({ jobId, runAt });
         this.delayedRunAt.set(jobId, runAt);
+        this.maybeCompactDelayedHeap();
     }
     /** Remove a delayed job (lazy removal from heap) */
     removeDelayed(jobId) {
-        if (this.delayedJobIds.has(jobId)) {
-            this.delayedJobIds.delete(jobId);
-            this.delayedRunAt.delete(jobId);
-            return true;
-        }
-        return false;
+        if (!this.delayedJobIds.delete(jobId))
+            return false;
+        this.delayedRunAt.delete(jobId);
+        this.maybeCompactDelayedHeap();
+        return true;
     }
     /**
      * Refresh delayed jobs that have become ready
@@ -127,8 +99,20 @@ export class TemporalManager {
             this.delayedRunAt.delete(top.jobId);
             count++;
         }
+        this.maybeCompactDelayedHeap();
         return count;
     }
+    maybeCompactDelayedHeap() {
+        const liveCount = this.delayedRunAt.size;
+        if (liveCount === 0) {
+            this.delayedHeap.clear();
+            return;
+        }
+        const staleCount = this.delayedHeap.size - liveCount;
+        if (staleCount < DELAYED_COMPACTION_MIN_STALE || staleCount < liveCount)
+            return;
+        this.delayedHeap.buildFrom(Array.from(this.delayedRunAt, ([jobId, runAt]) => ({ jobId, runAt })));
+    }
     /** Get delayed job count */
     get delayedCount() {
         return this.delayedJobIds.size;
@@ -143,8 +127,7 @@ export class TemporalManager {
     /** Clear all data */
     clear() {
         this.clearDelayed();
-        // Note: SkipList doesn't have a clear method, so we recreate by removing all
-        this.temporalIndex.removeAll(() => true);
+        this.temporalIndex.clear();
     }
     // ============ Debug Info ============
     /** Get internal structure sizes for memory debugging */

package/dist/domain/queue/waiterManager.d.ts

@@ -1,20 +1,18 @@
 /**
- * WaiterManager - Manages job availability notifications
- * Handles worker polling with timeout-based waiting
+ * WaiterManager - Manages queue-scoped job availability notifications.
+ * Handles worker polling with timeout-based waiting.
  */
 export declare class WaiterManager {
-    /** Waiter entries with cancellation flag for O(1) cleanup */
-    private readonly waiters;
-    /** Pending notification counter - incremented when notify() is called with no waiters */
-    private pendingNotifications;
-    /** Notify that multiple jobs are available - wakes up to `count` waiters */
+    private readonly queues;
+    private activeWaiters;
+    notify(queue?: string): void;
     notifyBatch(count: number): void;
-    /** Notify that jobs are available - wakes first non-cancelled waiter */
-    notify(): void;
-    /** Wait for a job to become available (with timeout) */
+    notifyBatch(queue: string, count: number): void;
     waitForJob(timeoutMs: number): Promise<void>;
-    /** Remove all cancelled waiters from the array */
-    private cleanupWaiters;
-    /** Current number of waiters */
+    waitForJob(queue: string, timeoutMs: number): Promise<void>;
+    private notifyQueue;
+    private createQueue;
+    private compact;
+    private deleteIdleQueue;
     get length(): number;
 }