bunqueue 2.8.32 → 2.8.33

package/dist/application/operations/queryOperations.js

@@ -123,6 +123,17 @@ export function getJobProgress(jobId, ctx) {
         return null;
     return { progress: job.progress, message: job.progressMessage };
 }
+/** Stable total order used by both in-memory collection and SQL queries. */
+function compareJobsByCreatedAt(a, b, asc) {
+    if (a.createdAt !== b.createdAt) {
+        return asc ? a.createdAt - b.createdAt : b.createdAt - a.createdAt;
+    }
+    if (a.id === b.id)
+        return 0;
+    if (asc)
+        return a.id < b.id ? -1 : 1;
+    return a.id > b.id ? -1 : 1;
+}
 /** Resolve job state from SQLite when jobIndex has no entry (post-restart recovery). */
 function resolveStateFromStorage(jobId, storage) {
     if (!storage)
@@ -197,29 +208,25 @@ export async function getJobState(jobId, ctx) {
     return 'unknown';
 }
 /** Collect completed jobs for a queue from index + storage */
-function collectCompletedJobs(queue, ctx, maxCollect) {
+function collectCompletedJobs(queue, ctx) {
     const jobs = [];
     for (const [jId, location] of ctx.jobIndex) {
         if (location.type === 'completed' && location.queueName === queue) {
             const job = ctx.storage?.getJob(jId) ?? ctx.completedJobsData?.get(jId) ?? null;
             if (job) {
                 jobs.push(job);
-                if (jobs.length >= maxCollect)
-                    break;
             }
         }
     }
     return jobs;
 }
 /** Collect active jobs for a queue across all processing shards */
-function collectActiveJobs(queue, shardIdx, ctx, maxCollect) {
+function collectActiveJobs(queue, shardIdx, ctx) {
     const jobs = [];
     // Own shard first (most likely location)
     for (const job of ctx.processingShards[shardIdx].values()) {
         if (job.queue === queue)
             jobs.push(job);
-        if (jobs.length >= maxCollect)
-            return jobs;
     }
     // Other shards
     for (let i = 0; i < ctx.shardCount; i++) {
@@ -228,20 +235,15 @@ function collectActiveJobs(queue, shardIdx, ctx, maxCollect) {
         for (const job of ctx.processingShards[i].values()) {
             if (job.queue === queue)
                 jobs.push(job);
-            if (jobs.length >= maxCollect)
-                return jobs;
         }
     }
     return jobs;
 }
 /** Collect waiting/delayed/prioritized jobs in a single pass */
-function collectTemporalJobs(shard, queue, needs, maxCollect) {
+function collectTemporalJobs(shard, queue, needs, now = Date.now()) {
     const { waiting: needWaiting, prioritized: needPrioritized, delayed: needDelayed } = needs;
-    const now = Date.now();
     const jobs = [];
     for (const j of shard.getQueue(queue).values()) {
-        if (jobs.length >= maxCollect)
-            break;
         const isDelayed = j.runAt > now;
         if (isDelayed && needDelayed) {
             jobs.push(j);
@@ -262,8 +264,7 @@ function tagState(jobs, state) {
     return jobs;
 }
 /** Tag temporal jobs with their actual state based on runAt/priority */
-function tagTemporalState(jobs) {
-    const now = Date.now();
+function tagTemporalState(jobs, now = Date.now()) {
     for (const j of jobs) {
         const isDelayed = j.runAt > now;
         j._state = isDelayed
@@ -274,19 +275,15 @@ function tagTemporalState(jobs) {
     }
 }
 /** Collect waiting-children jobs from deps and children maps */
-function collectWaitingChildrenFromShard(shard, queue, max) {
+function collectWaitingChildrenFromShard(shard, queue) {
     const wcJobs = [];
     for (const job of shard.waitingDeps.values()) {
         if (job.queue === queue)
             wcJobs.push(job);
-        if (wcJobs.length >= max)
-            return wcJobs;
     }
     for (const job of shard.waitingChildren.values()) {
         if (job.queue === queue)
             wcJobs.push(job);
-        if (wcJobs.length >= max)
-            return wcJobs;
     }
     return wcJobs;
 }
@@ -312,34 +309,33 @@ function resolveStateNeeds(states, paused) {
     };
 }
 /** When paused, the queue's ready jobs (waiting + prioritized) ARE the paused set (#92). */
-function collectPausedJobs(shard, queue, maxPerSource) {
-    const pausedJobs = collectTemporalJobs(shard, queue, { waiting: true, prioritized: true, delayed: false }, maxPerSource);
+function collectPausedJobs(shard, queue, now) {
+    const pausedJobs = collectTemporalJobs(shard, queue, { waiting: true, prioritized: true, delayed: false }, now);
     return tagState(pausedJobs, 'paused');
 }
-function collectJobsByState(queue, shardIdx, states, ctx, maxPerSource = Infinity) {
+function collectJobsByState(queue, shardIdx, states, ctx, now = Date.now()) {
     const shard = ctx.shards[shardIdx];
     const jobs = [];
     const need = resolveStateNeeds(states, shard.getState(queue).paused);
     if (need.waiting || need.prioritized || need.delayed) {
-        const temporal = collectTemporalJobs(shard, queue, { waiting: need.waiting, prioritized: need.prioritized, delayed: need.delayed }, maxPerSource);
-        tagTemporalState(temporal);
+        const temporal = collectTemporalJobs(shard, queue, { waiting: need.waiting, prioritized: need.prioritized, delayed: need.delayed }, now);
+        tagTemporalState(temporal, now);
         jobs.push(...temporal);
     }
     if (need.paused) {
-        jobs.push(...collectPausedJobs(shard, queue, maxPerSource));
+        jobs.push(...collectPausedJobs(shard, queue, now));
     }
     if (need.active) {
-        jobs.push(...tagState(collectActiveJobs(queue, shardIdx, ctx, maxPerSource), 'active'));
+        jobs.push(...tagState(collectActiveJobs(queue, shardIdx, ctx), 'active'));
     }
     if (need.failed) {
-        const dlq = shard.getDlq(queue);
-        jobs.push(...tagState(maxPerSource < dlq.length ? dlq.slice(0, maxPerSource) : dlq, 'failed'));
+        jobs.push(...tagState(shard.getDlq(queue), 'failed'));
     }
     if (need.completed) {
-        jobs.push(...tagState(collectCompletedJobs(queue, ctx, maxPerSource), 'completed'));
+        jobs.push(...tagState(collectCompletedJobs(queue, ctx), 'completed'));
     }
     if (need.waitingChildren) {
-        jobs.push(...tagState(collectWaitingChildrenFromShard(shard, queue, maxPerSource), 'waiting-children'));
+        jobs.push(...tagState(collectWaitingChildrenFromShard(shard, queue), 'waiting-children'));
     }
     return jobs;
 }
@@ -356,38 +352,34 @@ function collectWaitingChildrenJobs(shard, queue) {
     }
     return jobs;
 }
-/** Query SQLite with priority/waiting translation */
-function querySqliteWithPriority(storage, queue, sqlFilteredStates, opts) {
-    const hasPrioritized = sqlFilteredStates.includes('prioritized');
-    const hasWaiting = sqlFilteredStates.includes('waiting');
-    if (!hasPrioritized && !hasWaiting) {
-        if (sqlFilteredStates.length === 1) {
-            return storage.queryJobs(queue, { state: sqlFilteredStates[0], ...opts });
-        }
-        return storage.queryJobs(queue, { states: sqlFilteredStates, ...opts });
-    }
-    // Map 'prioritized' to 'waiting' for SQLite, then post-filter by priority
-    const sqlStates = sqlFilteredStates
-        .map((s) => (s === 'prioritized' ? 'waiting' : s))
-        .filter((s, i, arr) => arr.indexOf(s) === i);
-    const overFetchOpts = { ...opts, limit: opts.limit * 2 };
-    let jobs = sqlStates.length === 1
-        ? storage.queryJobs(queue, { state: sqlStates[0], ...overFetchOpts })
-        : storage.queryJobs(queue, { states: sqlStates, ...overFetchOpts });
-    if (hasPrioritized && !hasWaiting) {
-        jobs = jobs.filter((j) => j.priority > 0);
+/** Query SQLite after translating persisted pending rows to their logical state. */
+function querySqliteByLogicalState(storage, queue, sqlFilteredStates, opts) {
+    if (opts.excludedIds.size === 0) {
+        return storage.queryJobsByLogicalStates(queue, sqlFilteredStates, opts);
     }
-    else if (hasWaiting && !hasPrioritized) {
-        jobs = jobs.filter((j) => j.priority <= 0);
-    }
-    return jobs.slice(0, opts.limit);
+    // waitingDeps/waitingChildren rows remain persisted as waiting/delayed. Fetch
+    // enough rows to account for every possible exclusion, then paginate the
+    // logical result so parked jobs cannot create short or shifted pages.
+    const pageEnd = opts.offset + opts.limit;
+    const jobs = storage.queryJobsByLogicalStates(queue, sqlFilteredStates, {
+        limit: pageEnd + opts.excludedIds.size,
+        offset: 0,
+        asc: opts.asc,
+        now: opts.now,
+    });
+    return jobs.filter((job) => !opts.excludedIds.has(job.id)).slice(opts.offset, pageEnd);
 }
 /** Merge SQL rows with in-memory extras (each gathered from index 0), sort by
  *  createdAt, and paginate [start, end) once — so offset-unaware extras don't
  *  duplicate or drop rows across pages (#92). */
 function mergePage(sqlJobs, extras, start, end, asc) {
-    const merged = sqlJobs.concat(extras);
-    merged.sort((a, b) => (asc ? a.createdAt - b.createdAt : b.createdAt - a.createdAt));
+    const byId = new Map();
+    for (const job of sqlJobs)
+        byId.set(job.id, job);
+    for (const job of extras)
+        byId.set(job.id, job);
+    const merged = Array.from(byId.values());
+    merged.sort((a, b) => compareJobsByCreatedAt(a, b, asc));
     return merged.slice(start, end);
 }
 /** Get jobs from queue with filters */
@@ -401,10 +393,10 @@ export function getJobs(queue, shardIdx, options, ctx) {
                 : state
             : [state];
     const limit = end - start;
+    const now = Date.now();
     if (ctx.storage) {
         const shard = ctx.shards[shardIdx];
         const isPaused = shard.getState(queue).paused;
-        const maxPerSource = end + 1;
         // Derived sources are NOT offset-aware (the DLQ and the paused/waiting-children
         // views come from in-memory maps). When any contributes, we must gather [0, end)
         // from EVERY source, merge, sort, then slice [start, end) exactly once — pushing
@@ -417,6 +409,19 @@ export function getJobs(queue, shardIdx, options, ctx) {
             const all = ctx.storage.queryJobs(queue, { limit: end, offset: 0, asc });
             return mergePage(all, dlq, start, end, asc);
         }
+        // jobs-table states. A paused queue reports its waiting/prioritized jobs under
+        // 'paused', so they must not also surface in the waiting/prioritized lists (#92).
+        const sqlFilteredStates = states.filter((s) => s !== 'failed' &&
+            s !== 'waiting-children' &&
+            s !== 'paused' &&
+            !(isPaused && (s === 'waiting' || s === 'prioritized')));
+        // In-memory parked state is authoritative over the persisted row. Initial
+        // dependency jobs are stored as waiting/delayed, while parents moved after
+        // a pull can still be stored as active.
+        const parkedJobs = states.includes('waiting-children') || sqlFilteredStates.length > 0
+            ? collectWaitingChildrenJobs(shard, queue)
+            : [];
+        const parkedIds = new Set(parkedJobs.map((job) => job.id));
         // States with no jobs-table row are collected from in-memory sources:
         //  - 'failed'           -> DLQ (the failed job lives in the dlq table) (#92)
         //  - 'waiting-children' -> deps/children maps
@@ -426,40 +431,39 @@ export function getJobs(queue, shardIdx, options, ctx) {
             extras.push(...tagState(shard.getDlq(queue), 'failed'));
         }
         if (states.includes('waiting-children')) {
-            extras.push(...collectWaitingChildrenJobs(shard, queue));
+            extras.push(...tagState(parkedJobs, 'waiting-children'));
         }
         if (states.includes('paused') && isPaused) {
-            const pausedJobs = collectTemporalJobs(shard, queue, { waiting: true, prioritized: true, delayed: false }, maxPerSource);
+            const pausedJobs = collectTemporalJobs(shard, queue, { waiting: true, prioritized: true, delayed: false }, now);
             extras.push(...tagState(pausedJobs, 'paused'));
         }
-        // jobs-table states. A paused queue reports its waiting/prioritized jobs under
-        // 'paused', so they must not also surface in the waiting/prioritized lists (#92).
-        const sqlFilteredStates = states.filter((s) => s !== 'failed' &&
-            s !== 'waiting-children' &&
-            s !== 'paused' &&
-            !(isPaused && (s === 'waiting' || s === 'prioritized')));
         // Fast path: only jobs-table states, no derived sources — SQL paginates directly.
         if (extras.length === 0) {
             return sqlFilteredStates.length > 0
-                ? querySqliteWithPriority(ctx.storage, queue, sqlFilteredStates, {
+                ? querySqliteByLogicalState(ctx.storage, queue, sqlFilteredStates, {
                     limit,
                     offset: start,
                     asc,
+                    now,
+                    excludedIds: parkedIds,
                 })
                 : [];
         }
         const sqlJobs = sqlFilteredStates.length > 0
-            ? querySqliteWithPriority(ctx.storage, queue, sqlFilteredStates, {
+            ? querySqliteByLogicalState(ctx.storage, queue, sqlFilteredStates, {
                 limit: end,
                 offset: 0,
                 asc,
+                now,
+                excludedIds: parkedIds,
             })
             : [];
         return mergePage(sqlJobs, extras, start, end, asc);
     }
     // In-memory path (embedded mode only)
-    const maxPerSource = end + 1;
-    const jobs = collectJobsByState(queue, shardIdx, states, ctx, maxPerSource);
-    jobs.sort((a, b) => (asc ? a.createdAt - b.createdAt : b.createdAt - a.createdAt));
+    // Every source must be filtered before the global order is applied. Limiting
+    // an insertion-ordered Map here loses newer rows on descending pages.
+    const jobs = collectJobsByState(queue, shardIdx, states, ctx, now);
+    jobs.sort((a, b) => compareJobsByCreatedAt(a, b, asc));
     return jobs.slice(start, end);
 }

package/dist/application/queueManager.d.ts

@@ -341,24 +341,19 @@ export declare class QueueManager {
         paused: boolean;
         counts: {
             waiting: number;
+            prioritized: number;
             active: number;
             completed: number;
             failed: number;
             delayed: number;
         };
     }>;
+    /** Get counts for every registered queue with one global aggregation pass. */
+    getAllQueueJobCounts(): Map<string, statsMgr.QueueJobCounts>;
+    /** Aggregate a selected group of queues without repeating global scans. */
+    getQueueJobCountsBatch(queueNames: Iterable<string>): Map<string, statsMgr.QueueJobCounts>;
     /** Get job counts for a specific queue */
-    getQueueJobCounts(queueName: string): {
-        waiting: number;
-        prioritized: number;
-        delayed: number;
-        active: number;
-        completed: number;
-        failed: number;
-        'waiting-children': number;
-        totalCompleted: number;
-        totalFailed: number;
-    };
+    getQueueJobCounts(queueName: string): statsMgr.QueueJobCounts;
     getMemoryStats(): statsMgr.MemoryStats;
     /** Get storage health status (disk full detection) */
     getStorageStatus(): {

package/dist/application/queueManager.js

@@ -1321,7 +1321,7 @@ export class QueueManager {
                 queue.push(parentJob);
                 shard.incrementQueued(parentId, false, parentJob.createdAt, parentJob.queue, now);
                 this.jobIndex.set(parentId, { type: 'queue', shardIdx: idx, queueName: parentJob.queue });
-                shard.notify();
+                shard.notify(parentJob.queue);
                 promoted = true;
             }
         });
@@ -1435,7 +1435,7 @@ export class QueueManager {
                 shard.getQueue(parentJob.queue).push(parentJob);
                 shard.incrementQueued(parentId, false, parentJob.createdAt, parentJob.queue, now);
                 this.jobIndex.set(parentId, { type: 'queue', shardIdx: idx, queueName: parentJob.queue });
-                shard.notify();
+                shard.notify(parentJob.queue);
                 promoted = true;
             }
         });
@@ -1529,13 +1529,17 @@ export class QueueManager {
         const ctx = this.contextFactory.getStatsContext();
         const queues = this.listQueues();
         const result = [];
+        const countsByQueue = statsMgr.getAllQueueJobCounts(queues, ctx);
         for (const name of queues) {
-            const c = statsMgr.getQueueJobCounts(name, ctx);
+            const c = countsByQueue.get(name);
+            if (!c)
+                continue;
             result.push({
                 name,
                 paused: this.isPaused(name),
                 counts: {
                     waiting: c.waiting,
+                    prioritized: c.prioritized,
                     active: c.active,
                     completed: c.completed,
                     failed: c.failed,
@@ -1545,6 +1549,15 @@ export class QueueManager {
         }
         return result;
     }
+    /** Get counts for every registered queue with one global aggregation pass. */
+    getAllQueueJobCounts() {
+        return this.getQueueJobCountsBatch(this.queueNamesCache);
+    }
+    /** Aggregate a selected group of queues without repeating global scans. */
+    getQueueJobCountsBatch(queueNames) {
+        const ctx = this.contextFactory.getStatsContext();
+        return statsMgr.getAllQueueJobCounts(queueNames, ctx);
+    }
     /** Get job counts for a specific queue */
     getQueueJobCounts(queueName) {
         return statsMgr.getQueueJobCounts(queueName, this.contextFactory.getStatsContext());

package/dist/application/queueStatsAggregator.d.ts

@@ -0,0 +1,23 @@
+import type { StatsContext } from './types';
+export interface PerQueueStats {
+    waiting: number;
+    prioritized: number;
+    delayed: number;
+    active: number;
+    dlq: number;
+}
+export interface QueueJobCounts {
+    waiting: number;
+    prioritized: number;
+    delayed: number;
+    active: number;
+    completed: number;
+    failed: number;
+    'waiting-children': number;
+    totalCompleted: number;
+    totalFailed: number;
+}
+/** Aggregate all requested queues in one pass over every global collection. */
+export declare function getAllQueueJobCounts(queueNames: Iterable<string>, ctx: StatsContext): Map<string, QueueJobCounts>;
+export declare function getPerQueueStats(ctx: StatsContext, queueNames: Set<string>): Map<string, PerQueueStats>;
+export declare function getQueueJobCounts(queueName: string, ctx: StatsContext): QueueJobCounts;

package/dist/application/queueStatsAggregator.js

@@ -0,0 +1,80 @@
+import { SHARD_COUNT, shardIndex } from '../shared/hash';
+function createCounts(queue, ctx) {
+    const metrics = ctx.perQueueMetrics?.get(queue);
+    return {
+        waiting: 0,
+        prioritized: 0,
+        delayed: 0,
+        active: 0,
+        completed: 0,
+        failed: ctx.shards[shardIndex(queue)].getDlqCount(queue),
+        'waiting-children': 0,
+        totalCompleted: Number(metrics?.totalCompleted ?? 0n),
+        totalFailed: Number(metrics?.totalFailed ?? 0n),
+    };
+}
+/** Aggregate all requested queues in one pass over every global collection. */
+export function getAllQueueJobCounts(queueNames, ctx) {
+    const result = new Map();
+    const now = Date.now();
+    for (const name of queueNames) {
+        const counts = createCounts(name, ctx);
+        const queue = ctx.shards[shardIndex(name)].queues.get(name);
+        if (queue) {
+            for (const job of queue.values()) {
+                if (job.runAt > now)
+                    counts.delayed++;
+                else if (job.priority > 0)
+                    counts.prioritized++;
+                else
+                    counts.waiting++;
+            }
+        }
+        result.set(name, counts);
+    }
+    for (const processing of ctx.processingShards) {
+        for (const job of processing.values()) {
+            const counts = result.get(job.queue);
+            if (counts)
+                counts.active++;
+        }
+    }
+    for (const [jobId, location] of ctx.jobIndex) {
+        if (location.type !== 'completed' || !ctx.completedJobs.has(jobId))
+            continue;
+        const counts = result.get(location.queueName);
+        if (counts)
+            counts.completed++;
+    }
+    for (let i = 0; i < SHARD_COUNT; i++) {
+        const shard = ctx.shards[i];
+        for (const job of shard.waitingChildren.values()) {
+            const counts = result.get(job.queue);
+            if (counts)
+                counts['waiting-children']++;
+        }
+        for (const job of shard.waitingDeps.values()) {
+            const counts = result.get(job.queue);
+            if (counts)
+                counts['waiting-children']++;
+        }
+    }
+    return result;
+}
+export function getPerQueueStats(ctx, queueNames) {
+    const allCounts = getAllQueueJobCounts(queueNames, ctx);
+    const result = new Map();
+    for (const [name, counts] of allCounts) {
+        result.set(name, {
+            waiting: counts.waiting,
+            prioritized: counts.prioritized,
+            delayed: counts.delayed,
+            active: counts.active,
+            dlq: counts.failed,
+        });
+    }
+    return result;
+}
+export function getQueueJobCounts(queueName, ctx) {
+    return getAllQueueJobCounts([queueName], ctx).get(queueName) ?? createCounts(queueName, ctx);
+}

package/dist/application/statsManager.d.ts

@@ -3,6 +3,8 @@
  * Provides system metrics and memory compaction utilities
  */
 import type { StatsContext } from './types';
+export { getAllQueueJobCounts, getPerQueueStats, getQueueJobCounts, } from './queueStatsAggregator';
+export type { PerQueueStats, QueueJobCounts } from './queueStatsAggregator';
 export interface QueueStats {
     waiting: number;
     prioritized: number;
@@ -19,13 +21,6 @@ export interface QueueStats {
     cronJobs: number;
     cronPending: number;
 }
-export interface PerQueueStats {
-    waiting: number;
-    prioritized: number;
-    delayed: number;
-    active: number;
-    dlq: number;
-}
 export interface MemoryStats {
     jobIndex: number;
     completedJobs: number;
@@ -57,25 +52,6 @@ export declare function getStats(ctx: StatsContext, cronScheduler: {
  * Returns counts of entries in all major collections.
  */
 export declare function getMemoryStats(ctx: StatsContext): MemoryStats;
-/**
- * Get per-queue statistics by iterating shards to aggregate per queue name.
- * Uses shard hashing to look up each queue in its designated shard.
- */
-export declare function getPerQueueStats(ctx: StatsContext, queueNames: Set<string>): Map<string, PerQueueStats>;
-/**
- * Get job counts for a specific queue
- */
-export declare function getQueueJobCounts(queueName: string, ctx: StatsContext): {
-    waiting: number;
-    prioritized: number;
-    delayed: number;
-    active: number;
-    completed: number;
-    failed: number;
-    'waiting-children': number;
-    totalCompleted: number;
-    totalFailed: number;
-};
 /**
  * Force compact all collections to reduce memory usage.
  * Use after large batch operations or when memory pressure is high.