bunqueue 2.8.30 → 2.8.31

package/dist/application/operations/pull.js

@@ -47,18 +47,45 @@ function tryDequeueNextJob(shard, queue, now, ctx) {
     if (job.timeline.length < MAX_TIMELINE_ENTRIES) {
         job.timeline.push({ state: 'active', timestamp: now });
     }
+    // Make the queue -> processing transition atomic for observers. From the
+    // pop above, the job is no longer findable in the shard queue, so a
+    // jobIndex entry still saying 'queue' would make getJob/getJobState return
+    // a false null/'unknown' for an existing job (JOB -> null -> JOB(active)
+    // flicker), and cleanOrphanedJobIndex would even drop its index entry.
+    // Insert into processingShards and flip the index HERE, in the same
+    // synchronous critical section as the pop (the caller holds the shard
+    // write lock; JS is single-threaded, so no other task can interleave).
+    // Writing processingShards without its RWLock is safe: until this
+    // statement the index said 'queue', so no id-targeted critical section
+    // (ack/fail/discard operate only on ids they find as 'processing') can be
+    // mid-operation on this id; a sync Map.set cannot corrupt a paused
+    // critical section; and lock-free sync access to processingShards is
+    // established practice (stallDetection phases 1/2, collectActiveJobs,
+    // getJobProgress). Acquiring the async RWLock here would instead hold the
+    // hot shard write lock across an await.
+    const procIdx = processingShardIndex(job.id);
+    ctx.processingShards[procIdx].set(job.id, job);
+    ctx.jobIndex.set(job.id, { type: 'processing', shardIdx: procIdx });
     return { status: 'job', job };
 }
 /**
- * Move job to processing shard and broadcast
+ * Finish the handoff of a dequeued job: persist active state and broadcast.
+ *
+ * The job was ALREADY inserted into processingShards and jobIndex was flipped
+ * to 'processing' inside the dequeue critical section (tryDequeueNextJob),
+ * atomically with the queue pop, so readers never see a stale 'queue'
+ * location. Only the bookkeeping that may follow an await happens here.
+ *
+ * Returns false when the job is no longer in processingShards: a management
+ * op (discardJob, moveJobToDelayed, obliterate) claimed it between the
+ * dequeue and this call. The claimer owns the job now; the pull must NOT
+ * deliver it to a worker nor mark it active.
  */
-async function moveToProcessing(job, queue, ctx) {
+function finalizeProcessing(job, queue, ctx) {
     const procIdx = processingShardIndex(job.id);
+    if (!ctx.processingShards[procIdx].has(job.id))
+        return false;
     const now = Date.now();
-    await withWriteLock(ctx.processingLocks[procIdx], () => {
-        ctx.processingShards[procIdx].set(job.id, job);
-    });
-    ctx.jobIndex.set(job.id, { type: 'processing', shardIdx: procIdx });
     try {
         ctx.storage?.markActive(job.id, job.startedAt ?? now, job.timeline);
     }
@@ -74,74 +101,59 @@ async function moveToProcessing(job, queue, ctx) {
         jobId: job.id,
         timestamp: now,
     });
+    return true;
 }
 /**
- * Move multiple jobs to processing shards (optimized: groups by shard)
+ * Finish the handoff for a batch of dequeued jobs.
+ * Returns the jobs actually delivered (claimed jobs are skipped, see above).
  */
-async function moveToProcessingBatch(jobs, queue, ctx) {
-    // Group by processing shard for efficiency
-    const byProcShard = new Map();
+function finalizeProcessingBatch(jobs, queue, ctx) {
+    const delivered = [];
     for (const job of jobs) {
-        const procIdx = processingShardIndex(job.id);
-        const shardJobs = byProcShard.get(procIdx) ?? [];
-        if (shardJobs.length === 0)
-            byProcShard.set(procIdx, shardJobs);
-        shardJobs.push(job);
-    }
-    // Add to processing shards in parallel
-    const lockPromises = [];
-    for (const [procIdx, shardJobs] of byProcShard) {
-        lockPromises.push(withWriteLock(ctx.processingLocks[procIdx], () => {
-            for (const job of shardJobs) {
-                ctx.processingShards[procIdx].set(job.id, job);
-            }
-        }));
-    }
-    await Promise.all(lockPromises);
-    // Update indexes and broadcast
-    const now = Date.now();
-    for (const job of jobs) {
-        const procIdx = processingShardIndex(job.id);
-        ctx.jobIndex.set(job.id, { type: 'processing', shardIdx: procIdx });
-        try {
-            ctx.storage?.markActive(job.id, job.startedAt ?? now, job.timeline);
-        }
-        catch {
-            // Non-fatal: job is already in processingShards (in-memory source of truth).
-        }
-        ctx.totalPulled.value++;
-        throughputTracker.pullRate.increment();
-        ctx.broadcast({
-            eventType: 'pulled',
-            queue,
-            jobId: job.id,
-            timestamp: now,
-        });
+        if (finalizeProcessing(job, queue, ctx))
+            delivered.push(job);
     }
+    return delivered;
 }
 /**
- * Requeue a job that was popped from the priority queue but failed to move to processing.
- * Restores the job to the shard queue, removes it from processingShards, and notifies waiters.
+ * Requeue a job whose handoff to the worker failed after dequeue.
+ * The dequeue critical section moved the job to processingShards and flipped
+ * jobIndex to 'processing'; this restores both, keeping the same atomicity
+ * rule: the job becomes findable in the queue (push + index flip in ONE
+ * shard-lock section) BEFORE the processing entry is dropped, so a reader
+ * never sees a location that misses. Lock order shard -> processing matches
+ * the documented hierarchy.
  */
 async function requeueJob(job, queue, idx, ctx) {
-    // Remove from processingShards (may have been added before the failure)
     const procIdx = processingShardIndex(job.id);
-    await withWriteLock(ctx.processingLocks[procIdx], () => {
-        ctx.processingShards[procIdx].delete(job.id);
-    });
-    // Reset job state (was modified in tryDequeueNextJob)
-    job.startedAt = null;
-    // Push back to queue
+    let requeued = false;
     await withWriteLock(ctx.shardLocks[idx], () => {
+        // A management op (discardJob, moveJobToDelayed) may have claimed the job
+        // from processingShards after the failed handoff; it owns the job now.
+        if (!ctx.processingShards[procIdx].has(job.id))
+            return;
         const shard = ctx.shards[idx];
         if (job.groupId)
             shard.releaseGroup(queue, job.groupId);
         shard.releaseConcurrency(queue);
+        // Reset job state (was modified in tryDequeueNextJob)
+        job.startedAt = null;
         shard.getQueue(queue).push(job);
         shard.incrementQueued(job.id, false, job.createdAt, queue, job.runAt);
+        ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: queue });
         shard.notify();
+        requeued = true;
+    });
+    if (!requeued)
+        return;
+    await withWriteLock(ctx.processingLocks[procIdx], () => {
+        // A concurrent pull may have re-popped the job between the two lock
+        // sections and flipped the index back to 'processing'; do not delete the
+        // (re-inserted, same-object) entry out from under it.
+        if (ctx.jobIndex.get(job.id)?.type !== 'processing') {
+            ctx.processingShards[procIdx].delete(job.id);
+        }
     });
-    ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: queue });
 }
 /**
  * Pull next job from queue
@@ -153,14 +165,18 @@ export async function pullJob(queue, timeoutMs, ctx) {
     while (true) {
         const job = await tryPullFromShard(queue, idx, ctx);
         if (job) {
+            let delivered = false;
             try {
-                await moveToProcessing(job, queue, ctx);
+                delivered = finalizeProcessing(job, queue, ctx);
             }
             catch {
-                // Safety net: if moveToProcessing fails, push job back to prevent loss
+                // Safety net: if the handoff fails, push job back to prevent loss
                 await requeueJob(job, queue, idx, ctx);
                 return null;
             }
+            // Claimed by a management op during the handoff: try the next job
+            if (!delivered)
+                continue;
             latencyTracker.pull.observe((Bun.nanoseconds() - startNs) / 1e6);
             return job;
         }
@@ -216,21 +232,27 @@ export async function pullJobBatch(queue, count, timeoutMs, ctx) {
     while (true) {
         const jobs = await tryPullBatchFromShard(queue, idx, count, ctx);
         if (jobs.length > 0) {
+            let delivered;
             try {
-                await moveToProcessingBatch(jobs, queue, ctx);
+                delivered = finalizeProcessingBatch(jobs, queue, ctx);
             }
             catch {
-                // Safety net: push all jobs back to prevent loss
+                // Safety net: push all jobs back to prevent loss (requeueJob skips
+                // jobs a management op already claimed)
                 for (const job of jobs) {
                     await requeueJob(job, queue, idx, ctx);
                 }
                 return [];
             }
-            if (jobs.length > 1) {
-                ctx.dashboardEmit?.('batch:pulled', { queue, count: jobs.length });
+            if (delivered.length > 0) {
+                if (delivered.length > 1) {
+                    ctx.dashboardEmit?.('batch:pulled', { queue, count: delivered.length });
+                }
+                latencyTracker.pull.observe((Bun.nanoseconds() - startNs) / 1e6);
+                return delivered;
             }
-            latencyTracker.pull.observe((Bun.nanoseconds() - startNs) / 1e6);
-            return jobs;
+            // Every dequeued job was claimed by a management op: fall through to
+            // the timeout/wait logic below (nothing to deliver this round)
         }
         // No jobs available, check timeout
         // Cache Date.now() to avoid multiple syscalls per iteration

package/dist/application/operations/queryOperations.js

@@ -6,51 +6,75 @@ import { withReadLock } from '../../shared/lock';
 import { shardIndex } from '../../shared/hash';
 /** Get job by ID */
 export async function getJob(jobId, ctx) {
-    const location = ctx.jobIndex.get(jobId);
-    if (!location) {
-        // Fallback: jobIndex may not be populated after restart for completed/DLQ jobs.
-        // Consult SQLite directly so getJob survives recovery.
-        if (ctx.storage) {
-            const job = ctx.storage.getJob(jobId);
-            if (job)
-                return job;
-            const dlqEntry = ctx.storage.getDlqEntry(jobId);
-            if (dlqEntry)
-                return dlqEntry.job;
-        }
-        return ctx.completedJobsData.get(jobId) ?? null;
-    }
-    switch (location.type) {
-        case 'queue': {
-            return await withReadLock(ctx.shardLocks[location.shardIdx], () => {
-                const shard = ctx.shards[location.shardIdx];
-                return (shard.getQueue(location.queueName).find(jobId) ??
-                    shard.waitingDeps.get(jobId) ??
-                    shard.waitingChildren.get(jobId) ??
-                    null);
-            });
-        }
-        case 'processing': {
-            return await withReadLock(ctx.processingLocks[location.shardIdx], () => {
-                return ctx.processingShards[location.shardIdx].get(jobId) ?? null;
-            });
-        }
-        case 'completed':
-            return ctx.storage?.getJob(jobId) ?? ctx.completedJobsData.get(jobId) ?? null;
-        case 'dlq': {
+    // The location snapshot is only valid until the first await: while this
+    // reader waits on the shard read lock (writers have priority), a concurrent
+    // pull can pop the job and move it queue -> processing. The pull updates
+    // jobIndex atomically with the pop (tryDequeueNextJob), so on a miss the
+    // index re-read below is authoritative: an identity change means the job
+    // MOVED, and we chase the new location instead of returning a false null
+    // for a job that still exists (getJob(id) === null must be permanent for
+    // never-reused uuidv7 ids). Bounded: every extra pass requires a further
+    // state transition of this very job (queue -> active -> completed/removed),
+    // so 4 passes cover any realistic chase; index entries are always replaced
+    // (never mutated), which makes the identity comparison exact.
+    for (let pass = 0; pass < 4; pass++) {
+        const location = ctx.jobIndex.get(jobId);
+        if (!location) {
+            // Fallback: jobIndex may not be populated after restart for completed/DLQ jobs.
+            // Consult SQLite directly so getJob survives recovery.
             if (ctx.storage) {
-                const dlqEntry = ctx.storage.getDlqEntry(jobId);
-                if (dlqEntry)
-                    return dlqEntry.job;
                 const job = ctx.storage.getJob(jobId);
                 if (job)
                     return job;
+                const dlqEntry = ctx.storage.getDlqEntry(jobId);
+                if (dlqEntry)
+                    return dlqEntry.job;
             }
-            const dlqShardIdx = shardIndex(location.queueName);
-            const dlqJobs = ctx.shards[dlqShardIdx].getDlq(location.queueName);
-            return dlqJobs.find((j) => j.id === jobId) ?? null;
+            return ctx.completedJobsData.get(jobId) ?? null;
         }
+        let found = null;
+        switch (location.type) {
+            case 'queue': {
+                found = await withReadLock(ctx.shardLocks[location.shardIdx], () => {
+                    const shard = ctx.shards[location.shardIdx];
+                    return (shard.getQueue(location.queueName).find(jobId) ??
+                        shard.waitingDeps.get(jobId) ??
+                        shard.waitingChildren.get(jobId) ??
+                        null);
+                });
+                break;
+            }
+            case 'processing': {
+                found = await withReadLock(ctx.processingLocks[location.shardIdx], () => {
+                    return ctx.processingShards[location.shardIdx].get(jobId) ?? null;
+                });
+                break;
+            }
+            case 'completed':
+                found = ctx.storage?.getJob(jobId) ?? ctx.completedJobsData.get(jobId) ?? null;
+                break;
+            case 'dlq': {
+                if (ctx.storage) {
+                    const dlqEntry = ctx.storage.getDlqEntry(jobId);
+                    if (dlqEntry)
+                        return dlqEntry.job;
+                    const job = ctx.storage.getJob(jobId);
+                    if (job)
+                        return job;
+                }
+                const dlqShardIdx = shardIndex(location.queueName);
+                const dlqJobs = ctx.shards[dlqShardIdx].getDlq(location.queueName);
+                found = dlqJobs.find((j) => j.id === jobId) ?? null;
+                break;
+            }
+        }
+        if (found)
+            return found;
+        if (ctx.jobIndex.get(jobId) === location)
+            return null; // no move: genuine miss
+        // The job moved while we held the stale snapshot: chase it.
     }
+    return null;
 }
 /** Get job result */
 export function getJobResult(jobId, ctx) {
@@ -121,46 +145,56 @@ function resolveStateFromStorage(jobId, storage) {
 }
 /** Get job state by ID */
 export async function getJobState(jobId, ctx) {
-    const location = ctx.jobIndex.get(jobId);
-    // Check completed set first (fast path)
-    if (ctx.completedJobs.has(jobId)) {
-        return "completed" /* JobState.Completed */;
-    }
-    if (!location) {
-        return resolveStateFromStorage(jobId, ctx.storage);
-    }
-    switch (location.type) {
-        case 'queue': {
-            // Check if job is delayed, waiting, or waiting for children/deps
-            const result = await withReadLock(ctx.shardLocks[location.shardIdx], () => {
-                const shard = ctx.shards[location.shardIdx];
-                const queueJob = shard.getQueue(location.queueName).find(jobId);
-                if (queueJob)
-                    return { job: queueJob, waitingDeps: false, waitingChildren: false };
-                const depsJob = shard.waitingDeps.get(jobId);
-                if (depsJob)
-                    return { job: depsJob, waitingDeps: true, waitingChildren: false };
-                const childrenJob = shard.waitingChildren.get(jobId);
-                if (childrenJob)
-                    return { job: childrenJob, waitingDeps: false, waitingChildren: true };
-                return null;
-            });
-            if (!result)
-                return 'unknown';
-            if (result.waitingDeps || result.waitingChildren)
-                return 'waiting-children';
-            const now = Date.now();
-            if (result.job.runAt > now)
-                return "delayed" /* JobState.Delayed */;
-            return result.job.priority > 0 ? "prioritized" /* JobState.Prioritized */ : "waiting" /* JobState.Waiting */;
-        }
-        case 'processing':
-            return "active" /* JobState.Active */;
-        case 'completed':
+    // Same stale-snapshot chase as getJob: a 'queue' location read before the
+    // read-lock await may be outdated by a concurrent pull (queue -> processing)
+    // by the time the lookup runs. On a miss with a changed index entry, retry
+    // with the fresh location instead of reporting a false 'unknown'.
+    for (let pass = 0; pass < 4; pass++) {
+        const location = ctx.jobIndex.get(jobId);
+        // Check completed set first (fast path)
+        if (ctx.completedJobs.has(jobId)) {
             return "completed" /* JobState.Completed */;
-        case 'dlq':
-            return "failed" /* JobState.Failed */;
+        }
+        if (!location) {
+            return resolveStateFromStorage(jobId, ctx.storage);
+        }
+        switch (location.type) {
+            case 'queue': {
+                // Check if job is delayed, waiting, or waiting for children/deps
+                const result = await withReadLock(ctx.shardLocks[location.shardIdx], () => {
+                    const shard = ctx.shards[location.shardIdx];
+                    const queueJob = shard.getQueue(location.queueName).find(jobId);
+                    if (queueJob)
+                        return { job: queueJob, waitingDeps: false, waitingChildren: false };
+                    const depsJob = shard.waitingDeps.get(jobId);
+                    if (depsJob)
+                        return { job: depsJob, waitingDeps: true, waitingChildren: false };
+                    const childrenJob = shard.waitingChildren.get(jobId);
+                    if (childrenJob)
+                        return { job: childrenJob, waitingDeps: false, waitingChildren: true };
+                    return null;
+                });
+                if (!result) {
+                    if (ctx.jobIndex.get(jobId) === location)
+                        return 'unknown'; // no move
+                    break; // moved mid-lookup: chase the new location
+                }
+                if (result.waitingDeps || result.waitingChildren)
+                    return 'waiting-children';
+                const now = Date.now();
+                if (result.job.runAt > now)
+                    return "delayed" /* JobState.Delayed */;
+                return result.job.priority > 0 ? "prioritized" /* JobState.Prioritized */ : "waiting" /* JobState.Waiting */;
+            }
+            case 'processing':
+                return "active" /* JobState.Active */;
+            case 'completed':
+                return "completed" /* JobState.Completed */;
+            case 'dlq':
+                return "failed" /* JobState.Failed */;
+        }
     }
+    return 'unknown';
 }
 /** Collect completed jobs for a queue from index + storage */
 function collectCompletedJobs(queue, ctx, maxCollect) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bunqueue",
-  "version": "2.8.30",
+  "version": "2.8.31",
   "description": "High-performance job queue for Bun & AI agents. SQLite persistence, cron scheduling, priorities, retries, DLQ, webhooks, native MCP server. Zero external dependencies.",
   "type": "module",
   "main": "dist/main.js",