bunqueue 2.8.17 → 2.8.19

package/dist/application/backgroundTasks.js

@@ -319,6 +319,21 @@ export function recover(ctx) {
         }
         ctx.registerQueueName(queue);
     }
+    // === Restore queue control-state (#100) ===
+    // paused / rate-limit / concurrency live only in LimiterManager's in-memory
+    // Map; without this load every queue silently un-pauses and loses its limits
+    // on restart. Applied directly to the owning shard (in-memory only — these
+    // setters do not re-persist, so there is no write-back loop).
+    for (const qs of ctx.storage.loadQueueState()) {
+        const shard = ctx.shards[shardIndex(qs.name)];
+        if (qs.paused)
+            shard.pause(qs.name);
+        if (qs.rateLimit !== null)
+            shard.setRateLimit(qs.name, qs.rateLimit);
+        if (qs.concurrencyLimit !== null)
+            shard.setConcurrency(qs.name, qs.concurrencyLimit);
+        ctx.registerQueueName(qs.name);
+    }
     // === PHASE 3: Recover completed jobs ===
     // Required for clean('completed'), stats.completed, and in-memory lookups
     // on jobs that completed before a server restart (issue #84).

package/dist/application/queueManager.d.ts

@@ -88,6 +88,33 @@ export declare class QueueManager {
      * If the job was already requeued by the background lock expiration task, return silently.
      */
     private throwIfOwnershipConflict;
+    /**
+     * Issue #101 grace window: decide whether an ACK whose lock failed
+     * verification (because the TTL expired) should still be honored.
+     *
+     * Returns true ONLY when ALL hold:
+     *   1. the job is still in `processing`,
+     *   2. the lock entry's token still matches the presenting worker, and
+     *   3. the lock belongs to the CURRENT processing instance — its `createdAt`
+     *      is not older than the job's `startedAt`.
+     *
+     * Condition 3 is the re-lease guard. A lock-expiry re-lease (checkExpiredLocks)
+     * deletes the stale lock, so a new lease installs a NEW token and condition 2
+     * already fails. But the STALL path (stallDetection retry/moveToDlq) requeues
+     * the job WITHOUT deleting the lock — the original (now-expired) lock lingers
+     * with the original token. If another worker then re-pulls the job, its
+     * `startedAt` is reset to a newer time than the lingering lock's `createdAt`,
+     * so condition 3 fails and the timed-out worker's late ack is rejected
+     * (preventing a double-completion the skeptic confirmed). In the genuine #101
+     * case — the same worker finishing just after its lock expired, no re-pull —
+     * `startedAt` is unchanged and `createdAt >= startedAt`, so the grace is granted
+     * and the successful completion is recorded instead of being lost to a stall.
+     *
+     * Without this, a successful completion arriving just after lock expiry is
+     * rejected as "Invalid or expired lock token", the client drops it, and the
+     * job stalls to `failed` despite having been processed correctly.
+     */
+    private isExpiredButOwned;
     /** Check if a queued job was stall-retried (has been processed before). */
     private isStallRetried;
     /**
@@ -144,6 +171,13 @@ export declare class QueueManager {
     isPaused(queue: string): boolean;
     drain(queue: string): number;
     obliterate(queue: string): void;
+    /**
+     * Drop per-queue metadata that obliterate is responsible for reclaiming:
+     * cumulative metrics (keyed by name, never self-expiring) and the persisted
+     * control-state row (#100 — so a stale pause/limit can't resurrect on the
+     * next restart).
+     */
+    private purgeQueueMetadata;
     listQueues(): string[];
     private registerQueueName;
     private unregisterQueueName;
@@ -166,6 +200,12 @@ export declare class QueueManager {
     clearRateLimit(queue: string): void;
     setConcurrency(queue: string, limit: number): void;
     clearConcurrency(queue: string): void;
+    /**
+     * Issue #100: write-through the current control-state (paused / rate-limit /
+     * concurrency) to the `queue_state` table so it survives a server restart.
+     * Reads the post-mutation state from the owning shard and UPSERTs the row.
+     */
+    private persistQueueState;
     /** Get rate limit and concurrency limit for a queue */
     getQueueLimits(queue: string): {
         rateLimit: number | null;

package/dist/application/queueManager.js

@@ -276,7 +276,12 @@ export class QueueManager {
     }
     async ack(jobId, result, token) {
         const lockCtx = this.contextFactory.getLockContext();
-        if (token && !lockMgr.verifyLock(jobId, token, lockCtx)) {
+        if (token &&
+            !lockMgr.verifyLock(jobId, token, lockCtx) &&
+            !this.isExpiredButOwned(jobId, token, lockCtx)) {
+            // #101: if the lock is expired but still OURS and the job is still in
+            // `processing`, isExpiredButOwned() short-circuits this block so the
+            // completion falls through to ackJob() rather than being lost.
             this.throwIfOwnershipConflict(jobId, lockCtx);
             // No ownership conflict. If job is still in processing (dedup case
             // from Issue #33: lock removed but job still there), proceed with ACK.
@@ -334,7 +339,9 @@ export class QueueManager {
         if (tokens?.length === jobIds.length) {
             for (let i = 0; i < jobIds.length; i++) {
                 const t = tokens[i];
-                if (t && !lockMgr.verifyLock(jobIds[i], t, lockCtx)) {
+                if (t &&
+                    !lockMgr.verifyLock(jobIds[i], t, lockCtx) &&
+                    !this.isExpiredButOwned(jobIds[i], t, lockCtx)) {
                     this.throwIfOwnershipConflict(jobIds[i], lockCtx);
                     // Recover stall-retried job (#75): lock expired and job was
                     // re-queued by lock expiration or stall detection. Complete it
@@ -350,6 +357,8 @@ export class QueueManager {
                     }
                     continue;
                 }
+                // #101 grace window: an expired-but-still-ours lock on a still-processing
+                // job is accepted (isExpiredButOwned), not lost.
                 validJobIds.push(jobIds[i]);
                 if (validTokens)
                     validTokens.push(t);
@@ -376,7 +385,9 @@ export class QueueManager {
         const lockCtx = this.contextFactory.getLockContext();
         const validItems = [];
         for (const item of items) {
-            if (item.token && !lockMgr.verifyLock(item.id, item.token, lockCtx)) {
+            if (item.token &&
+                !lockMgr.verifyLock(item.id, item.token, lockCtx) &&
+                !this.isExpiredButOwned(item.id, item.token, lockCtx)) {
                 this.throwIfOwnershipConflict(item.id, lockCtx);
                 // Recover stall-retried job (#75): lock expired and job was
                 // re-queued by lock expiration or stall detection. Complete it
@@ -392,6 +403,9 @@ export class QueueManager {
                 }
                 continue;
             }
+            // #101 grace window (isExpiredButOwned true): the lock TTL elapsed while
+            // the handler ran, but the lock is still OURS and the job is still in
+            // `processing` — accept the completion instead of losing the work.
             validItems.push(item);
         }
         if (validItems.length > 0) {
@@ -436,6 +450,46 @@ export class QueueManager {
             throw new Error(`Invalid or expired lock token for job ${jobId}`);
         }
     }
+    /**
+     * Issue #101 grace window: decide whether an ACK whose lock failed
+     * verification (because the TTL expired) should still be honored.
+     *
+     * Returns true ONLY when ALL hold:
+     *   1. the job is still in `processing`,
+     *   2. the lock entry's token still matches the presenting worker, and
+     *   3. the lock belongs to the CURRENT processing instance — its `createdAt`
+     *      is not older than the job's `startedAt`.
+     *
+     * Condition 3 is the re-lease guard. A lock-expiry re-lease (checkExpiredLocks)
+     * deletes the stale lock, so a new lease installs a NEW token and condition 2
+     * already fails. But the STALL path (stallDetection retry/moveToDlq) requeues
+     * the job WITHOUT deleting the lock — the original (now-expired) lock lingers
+     * with the original token. If another worker then re-pulls the job, its
+     * `startedAt` is reset to a newer time than the lingering lock's `createdAt`,
+     * so condition 3 fails and the timed-out worker's late ack is rejected
+     * (preventing a double-completion the skeptic confirmed). In the genuine #101
+     * case — the same worker finishing just after its lock expired, no re-pull —
+     * `startedAt` is unchanged and `createdAt >= startedAt`, so the grace is granted
+     * and the successful completion is recorded instead of being lost to a stall.
+     *
+     * Without this, a successful completion arriving just after lock expiry is
+     * rejected as "Invalid or expired lock token", the client drops it, and the
+     * job stalls to `failed` despite having been processed correctly.
+     */
+    isExpiredButOwned(jobId, token, lockCtx) {
+        const loc = this.jobIndex.get(jobId);
+        if (loc?.type !== 'processing')
+            return false;
+        const lock = lockCtx.jobLocks.get(jobId);
+        if (lock?.token !== token)
+            return false;
+        // Re-lease guard: a re-pulled job has a startedAt newer than the lingering
+        // lock's createdAt → the lock no longer owns the current processing instance.
+        const job = this.processingShards[loc.shardIdx].get(jobId);
+        if (job && job.startedAt !== null && job.startedAt > lock.createdAt)
+            return false;
+        return true;
+    }
     /** Check if a queued job was stall-retried (has been processed before). */
     isStallRetried(jobId) {
         const loc = this.jobIndex.get(jobId);
@@ -653,6 +707,7 @@ export class QueueManager {
     // ============ Queue Control ============
     pause(queue) {
         queueControl.pauseQueue(queue, this.contextFactory.getQueueControlContext());
+        this.persistQueueState(queue);
         this.dashboardEmit?.('queue:paused', { queue });
         this.eventsManager.broadcast({
             eventType: "paused" /* EventType.Paused */,
@@ -663,6 +718,7 @@ export class QueueManager {
     }
     resume(queue) {
         queueControl.resumeQueue(queue, this.contextFactory.getQueueControlContext());
+        this.persistQueueState(queue);
         this.dashboardEmit?.('queue:resumed', { queue });
         this.eventsManager.broadcast({
             eventType: "resumed" /* EventType.Resumed */,
@@ -735,11 +791,21 @@ export class QueueManager {
         // their own; obliterate is the documented way to reclaim ALL state for a
         // queue, so drop its metrics entry too (prevents unbounded growth for
         // ephemeral/dynamically-named queues).
-        this.perQueueMetrics.delete(queue);
+        this.purgeQueueMetadata(queue);
         this.unregisterQueueName(queue);
         this.dashboardEmit?.('queue:obliterated', { queue });
         this.dashboardEmit?.('queue:removed', { queue });
     }
+    /**
+     * Drop per-queue metadata that obliterate is responsible for reclaiming:
+     * cumulative metrics (keyed by name, never self-expiring) and the persisted
+     * control-state row (#100 — so a stale pause/limit can't resurrect on the
+     * next restart).
+     */
+    purgeQueueMetadata(queue) {
+        this.perQueueMetrics.delete(queue);
+        this.storage?.deleteQueueState(queue);
+    }
     listQueues() {
         return Array.from(this.queueNamesCache);
     }
@@ -792,15 +858,38 @@ export class QueueManager {
     // ============ Rate Limiting ============
     setRateLimit(queue, limit) {
         this.shards[shardIndex(queue)].setRateLimit(queue, limit);
+        this.persistQueueState(queue);
     }
     clearRateLimit(queue) {
         this.shards[shardIndex(queue)].clearRateLimit(queue);
+        this.persistQueueState(queue);
     }
     setConcurrency(queue, limit) {
         this.shards[shardIndex(queue)].setConcurrency(queue, limit);
+        this.persistQueueState(queue);
     }
     clearConcurrency(queue) {
         this.shards[shardIndex(queue)].clearConcurrency(queue);
+        this.persistQueueState(queue);
+    }
+    /**
+     * Issue #100: write-through the current control-state (paused / rate-limit /
+     * concurrency) to the `queue_state` table so it survives a server restart.
+     * Reads the post-mutation state from the owning shard and UPSERTs the row.
+     */
+    persistQueueState(queue) {
+        if (!this.storage)
+            return;
+        const state = this.shards[shardIndex(queue)].getState(queue);
+        // When control-state returns fully to default (not paused, no limits), drop
+        // the row instead of persisting an all-default placeholder. Keeps the table
+        // free of noise rows for ephemeral queues that only ever call resume/clear*,
+        // and recovers identically (absent row → default state).
+        if (!state.paused && state.rateLimit === null && state.concurrencyLimit === null) {
+            this.storage.deleteQueueState(queue);
+            return;
+        }
+        this.storage.saveQueueState(queue, state.paused, state.rateLimit, state.concurrencyLimit);
     }
     /** Get rate limit and concurrency limit for a queue */
     getQueueLimits(queue) {

package/dist/client/worker/worker.d.ts

@@ -48,6 +48,7 @@ export declare class Worker<T = unknown, R = unknown> extends EventEmitter {
     private pendingJobs;
     private pendingJobsHead;
     private processingScheduled;
+    private pendingPull;
     private lastDrainedEmit;
     private stalledUnsubscribe;
     on(event: 'ready' | 'drained' | 'closed', listener: () => void): this;

package/dist/client/worker/worker.js

@@ -102,6 +102,9 @@ export class Worker extends EventEmitter {
     pendingJobs = [];
     pendingJobsHead = 0;
     processingScheduled = false; // Prevent multiple setImmediate calls
+    // Slots reserved by in-flight doPullBatch() calls (Issue #98). Subtracted from
+    // free slots so overlapping pulls see each other and do not over-lease.
+    pendingPull = 0;
     // Drained event tracking
     lastDrainedEmit = 0;
     // Stalled event subscription (BullMQ v5 compatible)
@@ -703,14 +706,46 @@ export class Worker extends EventEmitter {
         return null;
     }
     async doPullBatch() {
-        const slots = this.opts.concurrency - this.activeJobs;
+        // Issue #98: cap the LEASED count (running + buffered + in-flight pulls) at
+        // `concurrency`, not just the running count. The old `concurrency - activeJobs`
+        // was read once and the pull leases jobs on the broker across an await, so:
+        //   1. several concurrent finally->poll->tryProcess runs each read the same
+        //      stale count and each pull a full batch, and
+        //   2. a job just pulled by one run sits in `pendingJobs` (leased, counted by
+        //      the heartbeat) but not yet in `activeJobs`, so an overlapping pull does
+        //      not see it.
+        // Both leak: with concurrency=3 the worker ends up holding 5-6 jobs leased.
+        // `pulledJobIds.size` is the true leased count (active + buffered; a job is
+        // removed only on completion), and `pendingPull` reserves slots for pulls
+        // still in flight whose jobs are not yet registered.
+        //
+        // Exception — group pull-ahead: when a group limiter is set AND the buffer is
+        // non-empty here (this branch is reached only after getNextEligibleJob() found
+        // nothing runnable, so those buffered jobs are group-blocked), the worker must
+        // pull ahead to discover jobs from other, runnable groups — otherwise it would
+        // wedge on a buffer full of one blocked group. In that case the blocked
+        // buffered jobs are not counted (only the running ones are). This preserves the
+        // existing group behavior; the reported over-pull (no group limiter) always
+        // uses the strict leased cap.
+        const groupBlockedBuffer = this.groupLimiter !== null && this.pendingJobsHead < this.pendingJobs.length;
+        const leased = groupBlockedBuffer ? this.activeJobs : this.pulledJobIds.size;
+        const slots = this.opts.concurrency - leased - this.pendingPull;
         const batchSize = Math.min(this.opts.batchSize, slots, 1000);
         if (batchSize <= 0)
             return [];
         const config = this.getPullConfig();
-        return this.embedded
-            ? pullEmbedded(config, batchSize)
-            : pullTcp(config, this.tcp, batchSize, this._closing);
+        this.pendingPull += batchSize;
+        try {
+            return this.embedded
+                ? await pullEmbedded(config, batchSize)
+                : await pullTcp(config, this.tcp, batchSize, this._closing);
+        }
+        finally {
+            // Release the reservation. The pulled jobs are now registered/buffered (or
+            // the pull failed); either way the reservation has served its purpose for
+            // the duration of the in-flight pull.
+            this.pendingPull -= batchSize;
+        }
     }
     startJob(job, token) {
         const jobIdStr = String(job.id);

package/dist/infrastructure/persistence/sqlite.d.ts

@@ -175,6 +175,20 @@ export declare class SqliteStorage {
     deleteCron(name: string): void;
     /** Update cron job execution state (executions count and next run time) */
     updateCron(name: string, executions: number, nextRun: number): void;
+    /**
+     * Persist a queue's control-state (paused / rate-limit / concurrency) so it
+     * survives a server restart. Write-through on every pause/resume/limit change.
+     */
+    saveQueueState(name: string, paused: boolean, rateLimit: number | null, concurrencyLimit: number | null): void;
+    /** Load all persisted queue control-state rows (used by recover() on boot). */
+    loadQueueState(): Array<{
+        name: string;
+        paused: boolean;
+        rateLimit: number | null;
+        concurrencyLimit: number | null;
+    }>;
+    /** Drop a queue's persisted control-state (e.g. on obliterate). */
+    deleteQueueState(name: string): void;
     close(): void;
     getSize(): number;
 }

package/dist/infrastructure/persistence/sqlite.js

@@ -6,7 +6,7 @@
 import { Database } from 'bun:sqlite';
 import { createDlqEntry } from '../../domain/types/dlq';
 import { PRAGMA_SETTINGS, SCHEMA, MIGRATION_TABLE, SCHEMA_VERSION, MIGRATIONS } from './schema';
-import { prepareStatements } from './statements';
+import { prepareStatements, } from './statements';
 import { pack, unpack, rowToJob, reconstructDlqEntry } from './sqliteSerializer';
 import { BatchInsertManager, WriteBuffer } from './sqliteBatch';
 import { storageLog } from '../../shared/logger';
@@ -581,6 +581,34 @@ export class SqliteStorage {
             this.statements.get('updateCron').run(executions, nextRun, name);
         });
     }
+    // ============ Queue Control-State (#100) ============
+    /**
+     * Persist a queue's control-state (paused / rate-limit / concurrency) so it
+     * survives a server restart. Write-through on every pause/resume/limit change.
+     */
+    saveQueueState(name, paused, rateLimit, concurrencyLimit) {
+        this.safeWrite(() => {
+            this.statements
+                .get('upsertQueueState')
+                .run(name, paused ? 1 : 0, rateLimit, concurrencyLimit);
+        });
+    }
+    /** Load all persisted queue control-state rows (used by recover() on boot). */
+    loadQueueState() {
+        const rows = this.statements.get('loadQueueState').all();
+        return rows.map((row) => ({
+            name: row.name,
+            paused: row.paused === 1,
+            rateLimit: row.rate_limit,
+            concurrencyLimit: row.concurrency_limit,
+        }));
+    }
+    /** Drop a queue's persisted control-state (e.g. on obliterate). */
+    deleteQueueState(name) {
+        this.safeWrite(() => {
+            this.statements.get('deleteQueueState').run(name);
+        });
+    }
     // ============ Utilities ============
     close() {
         this.writeBuffer.stop();

package/dist/infrastructure/persistence/statements.d.ts

@@ -4,7 +4,7 @@
  */
 import type { Database } from 'bun:sqlite';
 /** Statement names */
-export type StatementName = 'insertJob' | 'updateJobState' | 'completeJob' | 'deleteJob' | 'deleteJobResult' | 'getJob' | 'insertResult' | 'getResult' | 'insertDlq' | 'loadDlq' | 'deleteDlqEntry' | 'clearDlqQueue' | 'insertCron' | 'updateCron';
+export type StatementName = 'insertJob' | 'updateJobState' | 'completeJob' | 'deleteJob' | 'deleteJobResult' | 'getJob' | 'insertResult' | 'getResult' | 'insertDlq' | 'loadDlq' | 'deleteDlqEntry' | 'clearDlqQueue' | 'insertCron' | 'updateCron' | 'upsertQueueState' | 'loadQueueState' | 'deleteQueueState';
 /** SQL statements */
 export declare const SQL_STATEMENTS: Record<StatementName, string>;
 /** Prepare all statements */
@@ -61,3 +61,10 @@ export interface DbCron {
     prevent_overlap: number;
     job_options: Uint8Array | null;
 }
+/** Database row type for queue control-state (#100) */
+export interface DbQueueState {
+    name: string;
+    paused: number;
+    rate_limit: number | null;
+    concurrency_limit: number | null;
+}

package/dist/infrastructure/persistence/statements.js

@@ -34,6 +34,10 @@ export const SQL_STATEMENTS = {
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `,
     updateCron: 'UPDATE cron_jobs SET executions = ?, next_run = ? WHERE name = ?',
+    // Queue control-state persistence (#100): paused / rate-limit / concurrency.
+    upsertQueueState: 'INSERT OR REPLACE INTO queue_state (name, paused, rate_limit, concurrency_limit) VALUES (?, ?, ?, ?)',
+    loadQueueState: 'SELECT name, paused, rate_limit, concurrency_limit FROM queue_state',
+    deleteQueueState: 'DELETE FROM queue_state WHERE name = ?',
 };
 /** Prepare all statements */
 export function prepareStatements(db) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bunqueue",
-  "version": "2.8.17",
+  "version": "2.8.19",
   "description": "High-performance job queue for Bun & AI agents. SQLite persistence, cron scheduling, priorities, retries, DLQ, webhooks, native MCP server. Zero external dependencies.",
   "type": "module",
   "main": "dist/main.js",