@trigger.dev/redis-worker 4.5.11 → 4.5.13

package/dist/index.js

@@ -9,7 +9,7 @@ import { calculateNextRetryDelay } from "@trigger.dev/core/v3";
 import { jumpHash, shutdownManager } from "@trigger.dev/core/v3/serverOnly";
 import { z } from "zod";
 import cronParser from "cron-parser";
-import { setInterval as setInterval$1 } from "node:timers/promises";
+import { setInterval as setInterval$1, setTimeout as setTimeout$1 } from "node:timers/promises";
 import seedrandom from "seedrandom";
 
 //#region \0rolldown/runtime.js
@@ -9636,22 +9636,28 @@ function createRedisClient(options, handlers) {
 }
 
 //#endregion
-//#region ../../node_modules/.pnpm/nanoid@5.1.2/node_modules/nanoid/url-alphabet/index.js
-const urlAlphabet = "useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict";
+//#region ../../node_modules/.pnpm/nanoid@5.1.16/node_modules/nanoid/url-alphabet/index.js
+let urlAlphabet = "useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict";
 
 //#endregion
-//#region ../../node_modules/.pnpm/nanoid@5.1.2/node_modules/nanoid/index.js
+//#region ../../node_modules/.pnpm/nanoid@5.1.16/node_modules/nanoid/index.js
 const POOL_SIZE_MULTIPLIER = 128;
 let pool;
 let poolOffset;
 function fillPool(bytes) {
-	if (!pool || pool.length < bytes) {
-		pool = Buffer.allocUnsafe(bytes * POOL_SIZE_MULTIPLIER);
-		webcrypto.getRandomValues(pool);
-		poolOffset = 0;
-	} else if (poolOffset + bytes > pool.length) {
-		webcrypto.getRandomValues(pool);
-		poolOffset = 0;
+	if (bytes < 0) throw new RangeError("Wrong ID size");
+	try {
+		if (!pool || pool.length < bytes) {
+			pool = Buffer.allocUnsafe(bytes * POOL_SIZE_MULTIPLIER);
+			webcrypto.getRandomValues(pool);
+			poolOffset = 0;
+		} else if (poolOffset + bytes > pool.length) {
+			webcrypto.getRandomValues(pool);
+			poolOffset = 0;
+		}
+	} catch (e) {
+		pool = void 0;
+		throw e;
 	}
 	poolOffset += bytes;
 }
@@ -12066,12 +12072,21 @@ var Worker = class Worker {
 		}
 		this.isShuttingDown = true;
 		this.logger.log("Shutting down worker loops...", { signal });
-		await Promise.race([Promise.all(this.workerLoops), Worker.delay(this.shutdownTimeoutMs).then(() => {
-			this.logger.error("Worker shutdown timed out", {
-				signal,
-				shutdownTimeoutMs: this.shutdownTimeoutMs
-			});
-		})]);
+		let shutdownDeadline;
+		const deadlinePromise = new Promise((resolve) => {
+			shutdownDeadline = setTimeout(() => {
+				this.logger.error("Worker shutdown timed out", {
+					signal,
+					shutdownTimeoutMs: this.shutdownTimeoutMs
+				});
+				resolve();
+			}, this.shutdownTimeoutMs);
+		});
+		try {
+			await Promise.race([Promise.all(this.workerLoops), deadlinePromise]);
+		} finally {
+			if (shutdownDeadline) clearTimeout(shutdownDeadline);
+		}
 		await this.subscriber?.unsubscribe();
 		await this.subscriber?.quit();
 		await this.queue.close();
@@ -12099,6 +12114,13 @@ function isAbortError(error) {
 //#endregion
 //#region src/fair-queue/concurrency.ts
 /**
+* Page size for iterating a concurrency set's members (SSCAN COUNT) and cap on how many
+* members one sweep script invocation checks, bounding both the snapshot reads and the
+* time the atomic Lua script can hold up Redis when a set has accumulated many leaked
+* members.
+*/
+const SWEEP_MEMBER_CHUNK_SIZE = 500;
+/**
 * ConcurrencyManager handles multi-level concurrency tracking and limiting.
 *
 * Features:
@@ -12112,12 +12134,17 @@ var ConcurrencyManager = class {
 	keys;
 	groups;
 	groupsByName;
+	logger;
 	constructor(options) {
 		this.options = options;
 		this.redis = createRedisClient(options.redis);
 		this.keys = options.keys;
 		this.groups = options.groups;
 		this.groupsByName = new Map(options.groups.map((g) => [g.name, g]));
+		this.logger = options.logger ?? {
+			debug: () => {},
+			error: () => {}
+		};
 		this.#registerCommands();
 	}
 	/**
@@ -12163,7 +12190,17 @@ var ConcurrencyManager = class {
 			const key = this.keys.concurrencyKey(group.name, groupId);
 			pipeline.srem(key, messageId);
 		}
-		await pipeline.exec();
+		this.#assertPipelineSucceeded(await pipeline.exec(), 1);
+	}
+	/**
+	* Throw if any command in a released pipeline failed. ioredis resolves `exec()` even when
+	* individual commands error, so an unchecked pipeline reports success while leaving the
+	* slot held, which strands it permanently once the caller drops the in-flight record.
+	*/
+	#assertPipelineSucceeded(results, messageCount) {
+		if (results === null) throw new Error(`Concurrency release pipeline for ${messageCount} message(s) was discarded without executing`);
+		const errors = results.map(([error]) => error).filter((error) => Boolean(error));
+		if (errors.length > 0) throw new Error(`Failed to release ${errors.length} of ${results?.length ?? 0} concurrency slot commands across ${messageCount} message(s): ${errors.map((error) => error.message).join("; ")}`);
 	}
 	/**
 	* Release concurrency slots for multiple messages in a single pipeline.
@@ -12177,7 +12214,69 @@ var ConcurrencyManager = class {
 			const key = this.keys.concurrencyKey(group.name, groupId);
 			pipeline.srem(key, messageId);
 		}
-		await pipeline.exec();
+		this.#assertPipelineSucceeded(await pipeline.exec(), messages.length);
+	}
+	/**
+	* Remove concurrency set members that no longer correspond to an in-flight message,
+	* healing slots leaked by failed releases. Scans every set of every group and, per
+	* member, atomically removes it unless the message id appears in one of the given
+	* in-flight data hashes. Sound because a message is registered in-flight before its
+	* slot is reserved, so at the moment of the atomic check a member with no in-flight
+	* record can only be a leak; if the message is about to be re-claimed, reserve simply
+	* re-adds the member.
+	*
+	* @param inflightDataKeys - The in-flight data hash keys for every shard. The sweep
+	*   refuses to run when this is empty, since with nowhere to look for running messages
+	*   every member would look orphaned and all concurrency accounting would be erased.
+	* @returns The message ids that were removed, and how many sets were checked
+	*/
+	async sweepOrphanedSlots(inflightDataKeys) {
+		if (inflightDataKeys.length === 0) {
+			this.logger.error("Refusing to sweep concurrency slots without any in-flight data keys: every member would look orphaned and all concurrency accounting would be erased");
+			return {
+				scannedSets: 0,
+				removed: []
+			};
+		}
+		const keyPrefix = this.options.redis.keyPrefix ?? "";
+		let scannedSets = 0;
+		const removed = [];
+		for (const group of this.groups) {
+			const pattern = `${keyPrefix}${this.keys.concurrencyKey(group.name, "*")}`;
+			let cursor = "0";
+			do {
+				const [nextCursor, foundKeys] = await this.redis.scan(cursor, "MATCH", pattern, "COUNT", 1e3);
+				cursor = nextCursor;
+				for (const fullKey of foundKeys) {
+					const key = keyPrefix && fullKey.startsWith(keyPrefix) ? fullKey.slice(keyPrefix.length) : fullKey;
+					try {
+						let sawMembers = false;
+						let memberCursor = "0";
+						do {
+							const [nextMemberCursor, page] = await this.redis.sscan(key, memberCursor, "COUNT", SWEEP_MEMBER_CHUNK_SIZE);
+							memberCursor = nextMemberCursor;
+							if (page.length === 0) continue;
+							sawMembers = true;
+							for (let i = 0; i < page.length; i += SWEEP_MEMBER_CHUNK_SIZE) {
+								const chunk = page.slice(i, i + SWEEP_MEMBER_CHUNK_SIZE);
+								const removedIds = await this.redis.removeOrphanedConcurrencySlots(1 + inflightDataKeys.length, [key, ...inflightDataKeys], ...chunk);
+								removed.push(...removedIds);
+							}
+						} while (memberCursor !== "0");
+						if (sawMembers) scannedSets++;
+					} catch (error) {
+						this.logger.error("Failed to sweep concurrency set, skipping it", {
+							key,
+							error: error instanceof Error ? error.message : String(error)
+						});
+					}
+				}
+			} while (cursor !== "0");
+		}
+		return {
+			scannedSets,
+			removed
+		};
 	}
 	/**
 	* Get current concurrency for a specific group.
@@ -12270,14 +12369,20 @@ var ConcurrencyManager = class {
 local numGroups = #KEYS
 local messageId = ARGV[1]
 
--- Check all groups first
+-- Check all groups first. A message that is already a member of a group's set passes
+-- that group's check: re-admitting it does not increase concurrency (SADD is a no-op),
+-- and counting its own leftover slot against it would let a message whose earlier
+-- release failed block its own retry forever.
 for i = 1, numGroups do
   local key = KEYS[i]
   local limit = tonumber(ARGV[1 + i])  -- Limits start at ARGV[2]
-  local current = redis.call('SCARD', key)
-  
-  if current >= limit then
-    return 0  -- At capacity
+
+  if redis.call('SISMEMBER', key, messageId) == 0 then
+    local current = redis.call('SCARD', key)
+
+    if current >= limit then
+      return 0  -- At capacity
+    end
   end
 end
 
@@ -12288,6 +12393,30 @@ for i = 1, numGroups do
 end
 
 return 1
+      ` });
+		this.redis.defineCommand("removeOrphanedConcurrencySlots", { lua: `
+local concurrencyKey = KEYS[1]
+local removedIds = {}
+
+for i = 1, #ARGV do
+  local messageId = ARGV[i]
+  local inflight = false
+
+  for j = 2, #KEYS do
+    if redis.call('HEXISTS', KEYS[j], messageId) == 1 then
+      inflight = true
+      break
+    end
+  end
+
+  if not inflight then
+    if redis.call('SREM', concurrencyKey, messageId) == 1 then
+      table.insert(removedIds, messageId)
+    end
+  end
+end
+
+return removedIds
       ` });
 	}
 };
@@ -13312,50 +13441,57 @@ var VisibilityManager = class {
 	*
 	* @param shardId - The shard to check
 	* @param getQueueKeys - Function to get queue keys for a queue ID
-	* @returns Array of reclaimed message info for concurrency release
+	* @param onBeforeRequeue - Hook invoked with the batch before any message is requeued,
+	*   used to free concurrency slots. A throw aborts the whole batch, leaving the
+	*   messages in-flight for the next reclaim tick.
+	* @returns Array of reclaimed message info
 	*/
-	async reclaimTimedOut(shardId, getQueueKeys) {
+	async reclaimTimedOut(shardId, getQueueKeys, onBeforeRequeue) {
 		const inflightKey = this.keys.inflightKey(shardId);
 		const inflightDataKey = this.keys.inflightDataKey(shardId);
 		const now = Date.now();
 		const timedOut = await this.redis.zrangebyscore(inflightKey, "-inf", now, "WITHSCORES", "LIMIT", 0, 100);
-		const reclaimedMessages = [];
+		const candidates = [];
 		for (let i = 0; i < timedOut.length; i += 2) {
 			const member = timedOut[i];
-			const _deadlineScore = timedOut[i + 1];
-			if (!member || !_deadlineScore) continue;
+			const deadlineScore = timedOut[i + 1];
+			if (!member || !deadlineScore) continue;
 			const { messageId, queueId } = this.#parseMember(member);
+			const dataJson = await this.redis.hget(inflightDataKey, messageId);
+			let storedMessage = null;
+			if (dataJson) try {
+				storedMessage = JSON.parse(dataJson);
+			} catch {}
+			if (!storedMessage) this.logger.error("Missing or corrupted message data during reclaim, using fallback", {
+				messageId,
+				queueId
+			});
+			candidates.push({
+				member,
+				deadlineScore,
+				storedMessage,
+				info: {
+					messageId,
+					queueId,
+					tenantId: storedMessage?.tenantId ?? this.keys.extractTenantId(queueId),
+					metadata: storedMessage?.metadata ?? {}
+				}
+			});
+		}
+		if (candidates.length === 0) return [];
+		if (onBeforeRequeue) await onBeforeRequeue(candidates.map((candidate) => candidate.info));
+		const reclaimedMessages = [];
+		for (const { member, deadlineScore, storedMessage, info } of candidates) {
+			const { messageId, queueId } = info;
 			const { queueKey, queueItemsKey, tenantQueueIndexKey, dispatchKey, tenantId } = getQueueKeys(queueId);
 			try {
-				const dataJson = await this.redis.hget(inflightDataKey, messageId);
-				let storedMessage = null;
-				if (dataJson) try {
-					storedMessage = JSON.parse(dataJson);
-				} catch {}
 				const score = storedMessage?.timestamp ?? now;
 				await this.redis.releaseMessage(inflightKey, inflightDataKey, queueKey, queueItemsKey, tenantQueueIndexKey, dispatchKey, member, messageId, score.toString(), queueId, "", tenantId);
-				if (storedMessage) reclaimedMessages.push({
-					messageId,
-					queueId,
-					tenantId: storedMessage.tenantId,
-					metadata: storedMessage.metadata
-				});
-				else {
-					this.logger.error("Missing or corrupted message data during reclaim, using fallback", {
-						messageId,
-						queueId
-					});
-					reclaimedMessages.push({
-						messageId,
-						queueId,
-						tenantId: this.keys.extractTenantId(queueId),
-						metadata: {}
-					});
-				}
+				reclaimedMessages.push(info);
 				this.logger.debug("Reclaimed timed-out message", {
 					messageId,
 					queueId,
-					deadline: _deadlineScore
+					deadline: deadlineScore
 				});
 			} catch (error) {
 				this.logger.error("Failed to reclaim message", {
@@ -13548,7 +13684,9 @@ local tenantId = ARGV[6]
 -- Get message data from in-flight
 local payload = redis.call('HGET', inflightDataKey, messageId)
 if not payload then
-  -- Message not in in-flight or already released
+  -- Message not in in-flight or already released. Drop the dangling in-flight
+  -- entry so it stops consuming a slot in every future reclaim scan.
+  redis.call('ZREM', inflightKey, member)
   return 0
 end
 
@@ -13558,14 +13696,15 @@ if updatedData and updatedData ~= "" then
   payload = updatedData
 end
 
+-- Add back to queue before dropping the in-flight record. Lua does not roll back on
+-- error, so removing first would lose the message outright if the queue write failed.
+redis.call('ZADD', queueKey, score, messageId)
+redis.call('HSET', queueItemsKey, messageId, payload)
+
 -- Remove from in-flight
 redis.call('ZREM', inflightKey, member)
 redis.call('HDEL', inflightDataKey, messageId)
 
--- Add back to queue
-redis.call('ZADD', queueKey, score, messageId)
-redis.call('HSET', queueItemsKey, messageId, payload)
-
 -- Update tenant queue index (Level 2) with queue's oldest message
 local oldest = redis.call('ZRANGE', queueKey, 0, 0, 'WITHSCORES')
 if #oldest >= 2 then
@@ -13610,15 +13749,20 @@ for i = 0, numMessages - 1 do
   -- Get message data from in-flight
   local payload = redis.call('HGET', inflightDataKey, messageId)
   if payload then
+    -- Add back to queue before dropping the in-flight record, so a failed queue write
+    -- cannot lose the message: Lua does not roll back already-applied commands.
+    redis.call('ZADD', queueKey, score, messageId)
+    redis.call('HSET', queueItemsKey, messageId, payload)
+
     -- Remove from in-flight
     redis.call('ZREM', inflightKey, member)
     redis.call('HDEL', inflightDataKey, messageId)
 
-    -- Add back to queue
-    redis.call('ZADD', queueKey, score, messageId)
-    redis.call('HSET', queueItemsKey, messageId, payload)
-
     releasedCount = releasedCount + 1
+  else
+    -- Dangling in-flight entry with no payload: drop it so it stops consuming
+    -- a slot in every future reclaim scan.
+    redis.call('ZREM', inflightKey, member)
   end
 end
 
@@ -13760,7 +13904,7 @@ var WorkerQueueManager = class {
 			throw error;
 		} finally {
 			if (cleanup && signal) signal.removeEventListener("abort", cleanup);
-			await blockingClient.quit().catch(() => {});
+			if (blockingClient.status !== "end") await blockingClient.quit().catch(() => {});
 		}
 	}
 	/**
@@ -14861,6 +15005,7 @@ var FairQueue = class {
 	visibilityTimeoutMs;
 	heartbeatIntervalMs;
 	reclaimIntervalMs;
+	reconcileIntervalMs;
 	workerQueueResolver;
 	batchClaimSize;
 	cooloffEnabled;
@@ -14877,6 +15022,7 @@ var FairQueue = class {
 	abortController;
 	masterQueueConsumerLoops = [];
 	reclaimLoop;
+	reconcileLoop;
 	queueDescriptorCache = /* @__PURE__ */ new Map();
 	tenantDispatch;
 	constructor(options) {
@@ -14896,6 +15042,7 @@ var FairQueue = class {
 		this.visibilityTimeoutMs = options.visibilityTimeoutMs ?? 3e4;
 		this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? this.visibilityTimeoutMs / 3;
 		this.reclaimIntervalMs = options.reclaimIntervalMs ?? 5e3;
+		this.reconcileIntervalMs = options.reconcileIntervalMs ?? 6e4;
 		this.workerQueueResolver = options.workerQueue.resolveWorkerQueue;
 		this.batchClaimSize = options.batchClaimSize ?? 10;
 		this.cooloffEnabled = options.cooloff?.enabled ?? true;
@@ -14934,7 +15081,11 @@ var FairQueue = class {
 		if (options.concurrencyGroups && options.concurrencyGroups.length > 0) this.concurrencyManager = new ConcurrencyManager({
 			redis: options.redis,
 			keys: options.keys,
-			groups: options.concurrencyGroups
+			groups: options.concurrencyGroups,
+			logger: {
+				debug: (msg, ctx) => this.logger.debug(msg, ctx),
+				error: (msg, ctx) => this.logger.error(msg, ctx)
+			}
 		});
 		this.visibilityManager = new VisibilityManager({
 			redis: options.redis,
@@ -15244,6 +15395,7 @@ var FairQueue = class {
 			this.masterQueueConsumerLoops.push(loop);
 		}
 		this.reclaimLoop = this.#runReclaimLoop();
+		if (this.concurrencyManager && this.reconcileIntervalMs > 0) this.reconcileLoop = this.#runReconcileLoop();
 		this.logger.info("FairQueue started", {
 			consumerCount: this.consumerCount,
 			shardCount: this.shardCount,
@@ -15257,9 +15409,14 @@ var FairQueue = class {
 		if (!this.isRunning) return;
 		this.isRunning = false;
 		this.abortController.abort();
-		await Promise.allSettled([...this.masterQueueConsumerLoops, this.reclaimLoop]);
+		await Promise.allSettled([
+			...this.masterQueueConsumerLoops,
+			this.reclaimLoop,
+			this.reconcileLoop
+		]);
 		this.masterQueueConsumerLoops = [];
 		this.reclaimLoop = void 0;
+		this.reconcileLoop = void 0;
 		this.logger.info("FairQueue stopped");
 	}
 	/**
@@ -15593,17 +15750,13 @@ var FairQueue = class {
 		if (dataJson) try {
 			storedMessage = JSON.parse(dataJson);
 		} catch {}
-		const descriptor = storedMessage ? this.queueDescriptorCache.get(queueId) ?? {
-			id: queueId,
-			tenantId: storedMessage.tenantId,
-			metadata: storedMessage.metadata ?? {}
-		} : {
+		const descriptor = this.queueDescriptorCache.get(queueId) ?? {
 			id: queueId,
-			tenantId: this.keys.extractTenantId(queueId),
-			metadata: {}
+			tenantId: storedMessage?.tenantId ?? this.keys.extractTenantId(queueId),
+			metadata: storedMessage?.metadata ?? {}
 		};
+		await this.#releaseConcurrencySafely(descriptor, messageId, "completeMessage");
 		await this.visibilityManager.complete(messageId, queueId);
-		if (this.concurrencyManager && storedMessage) await this.concurrencyManager.release(descriptor, messageId);
 		const { queueEmpty } = await this.#updateAllIndexesAfterDequeue(queueId, descriptor.tenantId);
 		if (queueEmpty) {
 			this.queueDescriptorCache.delete(queueId);
@@ -15632,26 +15785,59 @@ var FairQueue = class {
 		if (dataJson) try {
 			storedMessage = JSON.parse(dataJson);
 		} catch {}
-		const descriptor = storedMessage ? this.queueDescriptorCache.get(queueId) ?? {
-			id: queueId,
-			tenantId: storedMessage.tenantId,
-			metadata: storedMessage.metadata ?? {}
-		} : {
+		const descriptor = this.queueDescriptorCache.get(queueId) ?? {
 			id: queueId,
-			tenantId: this.keys.extractTenantId(queueId),
-			metadata: {}
+			tenantId: storedMessage?.tenantId ?? this.keys.extractTenantId(queueId),
+			metadata: storedMessage?.metadata ?? {}
 		};
+		await this.#releaseConcurrencySafely(descriptor, messageId, "releaseMessage");
 		const dispatchShardId = this.tenantDispatch.getShardForTenant(descriptor.tenantId);
 		const tenantQueueIndexKey = this.keys.tenantQueueIndexKey(descriptor.tenantId);
 		const dispatchKey = this.keys.dispatchKey(dispatchShardId);
 		await this.visibilityManager.release(messageId, queueId, queueKey, queueItemsKey, tenantQueueIndexKey, dispatchKey, descriptor.tenantId, Date.now());
-		if (this.concurrencyManager && storedMessage) await this.concurrencyManager.release(descriptor, messageId);
 		this.logger.debug("Message released", {
 			messageId,
 			queueId
 		});
 	}
 	/**
+	* Release a message's concurrency slots without letting a failure escape. Release is
+	* best-effort cleanup: the primary state transition (complete, retry, DLQ, requeue)
+	* must proceed even when the slot cannot be freed, because aborting the transition
+	* re-delivers or strands the message, while a leaked slot is recoverable — reserve()
+	* re-admits a message that already holds its own slot, and the reconcile loop removes
+	* members that are no longer in flight.
+	*/
+	async #releaseConcurrencySafely(descriptor, messageId, context) {
+		if (!this.concurrencyManager) return;
+		try {
+			await this.concurrencyManager.release(descriptor, messageId);
+		} catch (error) {
+			this.logger.error("Failed to release concurrency slot, leaving it for the reconciler", {
+				context,
+				messageId,
+				queueId: descriptor.id,
+				tenantId: descriptor.tenantId,
+				error: error instanceof Error ? error.message : String(error)
+			});
+		}
+	}
+	/**
+	* Release a concurrency slot for a message we can no longer describe, deriving the
+	* group from the queue id alone. Used on paths that bail out before the stored
+	* message is available, where the slot would otherwise be held with nothing left
+	* to reclaim it.
+	*/
+	async #releaseOrphanedConcurrency(messageId, queueId) {
+		if (!this.concurrencyManager) return;
+		const descriptor = this.queueDescriptorCache.get(queueId) ?? {
+			id: queueId,
+			tenantId: this.keys.extractTenantId(queueId),
+			metadata: {}
+		};
+		await this.#releaseConcurrencySafely(descriptor, messageId, "failMessage:missing-record");
+	}
+	/**
 	* Mark a message as failed. This will trigger retry logic if configured,
 	* or move the message to the dead letter queue.
 	*
@@ -15670,6 +15856,7 @@ var FairQueue = class {
 				messageId,
 				queueId
 			});
+			await this.#releaseOrphanedConcurrency(messageId, queueId);
 			return;
 		}
 		let storedMessage;
@@ -15680,6 +15867,7 @@ var FairQueue = class {
 				messageId,
 				queueId
 			});
+			await this.#releaseOrphanedConcurrency(messageId, queueId);
 			return;
 		}
 		const descriptor = this.queueDescriptorCache.get(queueId) ?? {
@@ -15699,10 +15887,10 @@ var FairQueue = class {
 					...storedMessage,
 					attempt: storedMessage.attempt + 1
 				};
+				await this.#releaseConcurrencySafely(descriptor, storedMessage.id, "retry");
 				const tenantQueueIndexKey = this.keys.tenantQueueIndexKey(descriptor.tenantId);
 				const dispatchKey = this.keys.dispatchKey(dispatchShardId);
 				await this.visibilityManager.release(storedMessage.id, queueId, queueKey, queueItemsKey, tenantQueueIndexKey, dispatchKey, descriptor.tenantId, Date.now() + nextDelay, JSON.stringify(updatedMessage));
-				if (this.concurrencyManager) await this.concurrencyManager.release(descriptor, storedMessage.id);
 				this.telemetry.recordRetry();
 				this.logger.debug("Message scheduled for retry", {
 					messageId: storedMessage.id,
@@ -15713,8 +15901,8 @@ var FairQueue = class {
 				return;
 			}
 		}
+		await this.#releaseConcurrencySafely(descriptor, storedMessage.id, "deadLetter");
 		await this.#moveToDeadLetterQueue(storedMessage, error?.message);
-		if (this.concurrencyManager) await this.concurrencyManager.release(descriptor, storedMessage.id);
 	}
 	async #moveToDeadLetterQueue(storedMessage, errorMessage) {
 		if (!this.deadLetterQueueEnabled) {
@@ -15763,9 +15951,37 @@ var FairQueue = class {
 			throw error;
 		}
 	}
+	/**
+	* Free every timed-out message's concurrency slot in one pipeline before the batch is
+	* requeued. A failed release never blocks the requeue: holding the message in-flight
+	* would stall it behind a persistent release failure, while the leaked slot self-heals
+	* when the message is re-admitted (reserve is idempotent for a message that already
+	* holds its own slot) or via the reconcile loop.
+	*/
+	async #releaseReclaimedConcurrency(messages) {
+		if (!this.concurrencyManager || messages.length === 0) return;
+		const descriptorFor = (message) => ({
+			id: message.queueId,
+			tenantId: message.tenantId,
+			metadata: message.metadata ?? {}
+		});
+		try {
+			await this.concurrencyManager.releaseBatch(messages.map((message) => ({
+				queue: descriptorFor(message),
+				messageId: message.messageId
+			})));
+			return;
+		} catch (error) {
+			this.logger.error("Batch concurrency release failed, retrying message by message", {
+				count: messages.length,
+				error: error instanceof Error ? error.message : String(error)
+			});
+		}
+		for (const message of messages) await this.#releaseConcurrencySafely(descriptorFor(message), message.messageId, "reclaim");
+	}
 	async #reclaimTimedOutMessages() {
 		let totalReclaimed = 0;
-		for (let shardId = 0; shardId < this.shardCount; shardId++) {
+		for (let shardId = 0; shardId < this.shardCount; shardId++) try {
 			const reclaimedMessages = await this.visibilityManager.reclaimTimedOut(shardId, (queueId) => {
 				const tenantId = this.keys.extractTenantId(queueId);
 				const dispatchShardId = this.tenantDispatch.getShardForTenant(tenantId);
@@ -15776,28 +15992,51 @@ var FairQueue = class {
 					dispatchKey: this.keys.dispatchKey(dispatchShardId),
 					tenantId
 				};
-			});
-			if (reclaimedMessages.length > 0) {
-				if (this.concurrencyManager) try {
-					await this.concurrencyManager.releaseBatch(reclaimedMessages.map((msg) => ({
-						queue: {
-							id: msg.queueId,
-							tenantId: msg.tenantId,
-							metadata: msg.metadata ?? {}
-						},
-						messageId: msg.messageId
-					})));
-				} catch (error) {
-					this.logger.error("Failed to release concurrency for reclaimed messages", {
-						count: reclaimedMessages.length,
-						error: error instanceof Error ? error.message : String(error)
-					});
-				}
-			}
+			}, this.#releaseReclaimedConcurrency.bind(this));
 			totalReclaimed += reclaimedMessages.length;
+		} catch (error) {
+			this.logger.error("Failed to reclaim shard, leaving messages in-flight for the next tick", {
+				shardId,
+				error: error instanceof Error ? error.message : String(error)
+			});
 		}
 		if (totalReclaimed > 0) this.logger.info("Reclaimed timed-out messages", { count: totalReclaimed });
 	}
+	/**
+	* The initial delay is jittered so instances sharing a Redis do not all sweep at once.
+	*/
+	async #runReconcileLoop() {
+		try {
+			await setTimeout$1(Math.random() * this.reconcileIntervalMs, null, { signal: this.abortController.signal });
+			for await (const _ of setInterval$1(this.reconcileIntervalMs, null, { signal: this.abortController.signal })) try {
+				await this.#reconcileConcurrency();
+			} catch (error) {
+				this.logger.error("Concurrency reconcile loop error", { error: error instanceof Error ? error.message : String(error) });
+			}
+		} catch (error) {
+			if (isAbortError(error)) {
+				this.logger.debug("Concurrency reconcile loop aborted");
+				return;
+			}
+			throw error;
+		}
+	}
+	/**
+	* Heal orphaned concurrency slots: remove any concurrency set member whose message is
+	* not registered in-flight on any shard. This is the recovery path for slots leaked by
+	* failed releases, which otherwise count against a tenant's limit forever and can
+	* wedge every queue the tenant owns.
+	*/
+	async #reconcileConcurrency() {
+		if (!this.concurrencyManager) return;
+		const inflightDataKeys = Array.from({ length: this.shardCount }, (_, shardId) => this.keys.inflightDataKey(shardId));
+		const { scannedSets, removed } = await this.concurrencyManager.sweepOrphanedSlots(inflightDataKeys);
+		if (removed.length > 0) this.logger.info("Reconciled orphaned concurrency slots", {
+			count: removed.length,
+			scannedSets,
+			messageIds: removed.slice(0, 50)
+		});
+	}
 	#isInCooloff(queueId) {
 		const state = this.queueCooloffStates.get(queueId);
 		if (!state) return false;