@trigger.dev/redis-worker 4.3.2 → 4.4.0

package/dist/index.d.cts

@@ -449,6 +449,20 @@ interface ConcurrencyCheckResult {
     /** If not allowed, which group is blocking */
     blockedBy?: ConcurrencyState;
 }
+/**
+ * Information about a reclaimed message from visibility timeout.
+ * Used to release concurrency after a message is returned to the queue.
+ */
+interface ReclaimedMessageInfo {
+    /** Message ID */
+    messageId: string;
+    /** Queue ID */
+    queueId: string;
+    /** Tenant ID for concurrency release */
+    tenantId: string;
+    /** Additional metadata for concurrency group extraction */
+    metadata?: Record<string, unknown>;
+}
 /**
  * Queues grouped by tenant for the scheduler.
  */
@@ -994,6 +1008,14 @@ declare class ConcurrencyManager {
      * Release concurrency slots for a message across all groups.
      */
     release(queue: QueueDescriptor, messageId: string): Promise<void>;
+    /**
+     * Release concurrency slots for multiple messages in a single pipeline.
+     * More efficient than calling release() multiple times.
+     */
+    releaseBatch(messages: Array<{
+        queue: QueueDescriptor;
+        messageId: string;
+    }>): Promise<void>;
     /**
      * Get current concurrency for a specific group.
      */
@@ -1151,13 +1173,13 @@ declare class VisibilityManager {
      *
      * @param shardId - The shard to check
      * @param getQueueKeys - Function to get queue keys for a queue ID
-     * @returns Number of messages reclaimed
+     * @returns Array of reclaimed message info for concurrency release
      */
     reclaimTimedOut(shardId: number, getQueueKeys: (queueId: string) => {
         queueKey: string;
         queueItemsKey: string;
         masterQueueKey: string;
-    }): Promise<number>;
+    }): Promise<ReclaimedMessageInfo[]>;
     /**
      * Get all in-flight messages for a shard.
      */
@@ -1923,4 +1945,4 @@ declare module "@internal/redis" {
     }
 }
 
-export { type AnyMessageCatalog, type AnyQueueItem, BaseScheduler, BatchedSpanManager, type BatchedSpanManagerOptions, CallbackFairQueueKeyProducer, type ClaimResult, type ConcurrencyCheckResult, type ConcurrencyGroupConfig, ConcurrencyManager, type ConcurrencyManagerOptions, type ConcurrencyState, type ConsumerLoopState, type CooloffOptions, CronSchema, CustomRetry, DRRScheduler, type DRRSchedulerConfig, type DeadLetterMessage, DefaultFairQueueKeyProducer, type EnqueueBatchOptions, type EnqueueOptions, ExponentialBackoffRetry, FairQueue, FairQueueAttributes, type FairQueueKeyProducer, type FairQueueMetrics, type FairQueueOptions, FairQueueTelemetry, type FairScheduler, FixedDelayRetry, type GlobalRateLimiter, ImmediateRetry, type InFlightMessage, type JobHandler, type JobHandlerParams, LinearBackoffRetry, MasterQueue, type MasterQueueOptions, type MessageCatalogKey, type MessageCatalogSchema, type MessageCatalogValue, type MessageHandler, type MessageHandlerContext, MessagingAttributes, NoRetry, NoopScheduler, type QueueCooloffState, type QueueDescriptor, type QueueItem, type QueueMessage, type QueueWithScore, type RetryOptions, type RetryStrategy, RoundRobinScheduler, type SchedulerContext, SimpleQueue, type StoredMessage, type TelemetryOptions, type TenantQueues, VisibilityManager, type VisibilityManagerOptions, WeightedScheduler, type WeightedSchedulerBiases, type WeightedSchedulerConfig, Worker, type WorkerCatalog, type WorkerConcurrencyOptions, WorkerQueueManager, type WorkerQueueManagerOptions, type WorkerQueueOptions, createDefaultRetryStrategy, defaultRetryOptions, isAbortError, noopTelemetry };
+export { type AnyMessageCatalog, type AnyQueueItem, BaseScheduler, BatchedSpanManager, type BatchedSpanManagerOptions, CallbackFairQueueKeyProducer, type ClaimResult, type ConcurrencyCheckResult, type ConcurrencyGroupConfig, ConcurrencyManager, type ConcurrencyManagerOptions, type ConcurrencyState, type ConsumerLoopState, type CooloffOptions, CronSchema, CustomRetry, DRRScheduler, type DRRSchedulerConfig, type DeadLetterMessage, DefaultFairQueueKeyProducer, type EnqueueBatchOptions, type EnqueueOptions, ExponentialBackoffRetry, FairQueue, FairQueueAttributes, type FairQueueKeyProducer, type FairQueueMetrics, type FairQueueOptions, FairQueueTelemetry, type FairScheduler, FixedDelayRetry, type GlobalRateLimiter, ImmediateRetry, type InFlightMessage, type JobHandler, type JobHandlerParams, LinearBackoffRetry, MasterQueue, type MasterQueueOptions, type MessageCatalogKey, type MessageCatalogSchema, type MessageCatalogValue, type MessageHandler, type MessageHandlerContext, MessagingAttributes, NoRetry, NoopScheduler, type QueueCooloffState, type QueueDescriptor, type QueueItem, type QueueMessage, type QueueWithScore, type ReclaimedMessageInfo, type RetryOptions, type RetryStrategy, RoundRobinScheduler, type SchedulerContext, SimpleQueue, type StoredMessage, type TelemetryOptions, type TenantQueues, VisibilityManager, type VisibilityManagerOptions, WeightedScheduler, type WeightedSchedulerBiases, type WeightedSchedulerConfig, Worker, type WorkerCatalog, type WorkerConcurrencyOptions, WorkerQueueManager, type WorkerQueueManagerOptions, type WorkerQueueOptions, createDefaultRetryStrategy, defaultRetryOptions, isAbortError, noopTelemetry };

package/dist/index.d.ts

@@ -449,6 +449,20 @@ interface ConcurrencyCheckResult {
     /** If not allowed, which group is blocking */
     blockedBy?: ConcurrencyState;
 }
+/**
+ * Information about a reclaimed message from visibility timeout.
+ * Used to release concurrency after a message is returned to the queue.
+ */
+interface ReclaimedMessageInfo {
+    /** Message ID */
+    messageId: string;
+    /** Queue ID */
+    queueId: string;
+    /** Tenant ID for concurrency release */
+    tenantId: string;
+    /** Additional metadata for concurrency group extraction */
+    metadata?: Record<string, unknown>;
+}
 /**
  * Queues grouped by tenant for the scheduler.
  */
@@ -994,6 +1008,14 @@ declare class ConcurrencyManager {
      * Release concurrency slots for a message across all groups.
      */
     release(queue: QueueDescriptor, messageId: string): Promise<void>;
+    /**
+     * Release concurrency slots for multiple messages in a single pipeline.
+     * More efficient than calling release() multiple times.
+     */
+    releaseBatch(messages: Array<{
+        queue: QueueDescriptor;
+        messageId: string;
+    }>): Promise<void>;
     /**
      * Get current concurrency for a specific group.
      */
@@ -1151,13 +1173,13 @@ declare class VisibilityManager {
      *
      * @param shardId - The shard to check
      * @param getQueueKeys - Function to get queue keys for a queue ID
-     * @returns Number of messages reclaimed
+     * @returns Array of reclaimed message info for concurrency release
      */
     reclaimTimedOut(shardId: number, getQueueKeys: (queueId: string) => {
         queueKey: string;
         queueItemsKey: string;
         masterQueueKey: string;
-    }): Promise<number>;
+    }): Promise<ReclaimedMessageInfo[]>;
     /**
      * Get all in-flight messages for a shard.
      */
@@ -1923,4 +1945,4 @@ declare module "@internal/redis" {
     }
 }
 
-export { type AnyMessageCatalog, type AnyQueueItem, BaseScheduler, BatchedSpanManager, type BatchedSpanManagerOptions, CallbackFairQueueKeyProducer, type ClaimResult, type ConcurrencyCheckResult, type ConcurrencyGroupConfig, ConcurrencyManager, type ConcurrencyManagerOptions, type ConcurrencyState, type ConsumerLoopState, type CooloffOptions, CronSchema, CustomRetry, DRRScheduler, type DRRSchedulerConfig, type DeadLetterMessage, DefaultFairQueueKeyProducer, type EnqueueBatchOptions, type EnqueueOptions, ExponentialBackoffRetry, FairQueue, FairQueueAttributes, type FairQueueKeyProducer, type FairQueueMetrics, type FairQueueOptions, FairQueueTelemetry, type FairScheduler, FixedDelayRetry, type GlobalRateLimiter, ImmediateRetry, type InFlightMessage, type JobHandler, type JobHandlerParams, LinearBackoffRetry, MasterQueue, type MasterQueueOptions, type MessageCatalogKey, type MessageCatalogSchema, type MessageCatalogValue, type MessageHandler, type MessageHandlerContext, MessagingAttributes, NoRetry, NoopScheduler, type QueueCooloffState, type QueueDescriptor, type QueueItem, type QueueMessage, type QueueWithScore, type RetryOptions, type RetryStrategy, RoundRobinScheduler, type SchedulerContext, SimpleQueue, type StoredMessage, type TelemetryOptions, type TenantQueues, VisibilityManager, type VisibilityManagerOptions, WeightedScheduler, type WeightedSchedulerBiases, type WeightedSchedulerConfig, Worker, type WorkerCatalog, type WorkerConcurrencyOptions, WorkerQueueManager, type WorkerQueueManagerOptions, type WorkerQueueOptions, createDefaultRetryStrategy, defaultRetryOptions, isAbortError, noopTelemetry };
+export { type AnyMessageCatalog, type AnyQueueItem, BaseScheduler, BatchedSpanManager, type BatchedSpanManagerOptions, CallbackFairQueueKeyProducer, type ClaimResult, type ConcurrencyCheckResult, type ConcurrencyGroupConfig, ConcurrencyManager, type ConcurrencyManagerOptions, type ConcurrencyState, type ConsumerLoopState, type CooloffOptions, CronSchema, CustomRetry, DRRScheduler, type DRRSchedulerConfig, type DeadLetterMessage, DefaultFairQueueKeyProducer, type EnqueueBatchOptions, type EnqueueOptions, ExponentialBackoffRetry, FairQueue, FairQueueAttributes, type FairQueueKeyProducer, type FairQueueMetrics, type FairQueueOptions, FairQueueTelemetry, type FairScheduler, FixedDelayRetry, type GlobalRateLimiter, ImmediateRetry, type InFlightMessage, type JobHandler, type JobHandlerParams, LinearBackoffRetry, MasterQueue, type MasterQueueOptions, type MessageCatalogKey, type MessageCatalogSchema, type MessageCatalogValue, type MessageHandler, type MessageHandlerContext, MessagingAttributes, NoRetry, NoopScheduler, type QueueCooloffState, type QueueDescriptor, type QueueItem, type QueueMessage, type QueueWithScore, type ReclaimedMessageInfo, type RetryOptions, type RetryStrategy, RoundRobinScheduler, type SchedulerContext, SimpleQueue, type StoredMessage, type TelemetryOptions, type TenantQueues, VisibilityManager, type VisibilityManagerOptions, WeightedScheduler, type WeightedSchedulerBiases, type WeightedSchedulerConfig, Worker, type WorkerCatalog, type WorkerConcurrencyOptions, WorkerQueueManager, type WorkerQueueManagerOptions, type WorkerQueueOptions, createDefaultRetryStrategy, defaultRetryOptions, isAbortError, noopTelemetry };

package/dist/index.js

@@ -11800,6 +11800,24 @@ var ConcurrencyManager = class {
     }
     await pipeline.exec();
   }
+  /**
+   * Release concurrency slots for multiple messages in a single pipeline.
+   * More efficient than calling release() multiple times.
+   */
+  async releaseBatch(messages) {
+    if (messages.length === 0) {
+      return;
+    }
+    const pipeline = this.redis.pipeline();
+    for (const { queue, messageId } of messages) {
+      for (const group of this.groups) {
+        const groupId = group.extractGroupId(queue);
+        const key = this.keys.concurrencyKey(group.name, groupId);
+        pipeline.srem(key, messageId);
+      }
+    }
+    await pipeline.exec();
+  }
   /**
    * Get current concurrency for a specific group.
    */
@@ -12894,7 +12912,7 @@ var VisibilityManager = class {
    *
    * @param shardId - The shard to check
    * @param getQueueKeys - Function to get queue keys for a queue ID
-   * @returns Number of messages reclaimed
+   * @returns Array of reclaimed message info for concurrency release
    */
   async reclaimTimedOut(shardId, getQueueKeys) {
     const inflightKey = this.keys.inflightKey(shardId);
@@ -12910,17 +12928,25 @@ var VisibilityManager = class {
       100
       // Process in batches
     );
-    let reclaimed = 0;
+    const reclaimedMessages = [];
     for (let i = 0; i < timedOut.length; i += 2) {
       const member = timedOut[i];
-      const originalScore = timedOut[i + 1];
-      if (!member || !originalScore) {
+      const _deadlineScore = timedOut[i + 1];
+      if (!member || !_deadlineScore) {
         continue;
       }
       const { messageId, queueId } = this.#parseMember(member);
       const { queueKey, queueItemsKey, masterQueueKey } = getQueueKeys(queueId);
       try {
-        const score = parseFloat(originalScore) || now;
+        const dataJson = await this.redis.hget(inflightDataKey, messageId);
+        let storedMessage = null;
+        if (dataJson) {
+          try {
+            storedMessage = JSON.parse(dataJson);
+          } catch {
+          }
+        }
+        const score = storedMessage?.timestamp ?? now;
         await this.redis.releaseMessage(
           inflightKey,
           inflightDataKey,
@@ -12932,11 +12958,29 @@ var VisibilityManager = class {
           score.toString(),
           queueId
         );
-        reclaimed++;
+        if (storedMessage) {
+          reclaimedMessages.push({
+            messageId,
+            queueId,
+            tenantId: storedMessage.tenantId,
+            metadata: storedMessage.metadata
+          });
+        } else {
+          this.logger.error("Missing or corrupted message data during reclaim, using fallback", {
+            messageId,
+            queueId
+          });
+          reclaimedMessages.push({
+            messageId,
+            queueId,
+            tenantId: this.keys.extractTenantId(queueId),
+            metadata: {}
+          });
+        }
         this.logger.debug("Reclaimed timed-out message", {
           messageId,
           queueId,
-          originalScore
+          deadline: _deadlineScore
         });
       } catch (error) {
         this.logger.error("Failed to reclaim message", {
@@ -12946,7 +12990,7 @@ var VisibilityManager = class {
         });
       }
     }
-    return reclaimed;
+    return reclaimedMessages;
   }
   /**
    * Get all in-flight messages for a shard.
@@ -15349,12 +15393,31 @@ var FairQueue = class {
   async #reclaimTimedOutMessages() {
     let totalReclaimed = 0;
     for (let shardId = 0; shardId < this.shardCount; shardId++) {
-      const reclaimed = await this.visibilityManager.reclaimTimedOut(shardId, (queueId) => ({
+      const reclaimedMessages = await this.visibilityManager.reclaimTimedOut(shardId, (queueId) => ({
         queueKey: this.keys.queueKey(queueId),
         queueItemsKey: this.keys.queueItemsKey(queueId),
         masterQueueKey: this.keys.masterQueueKey(this.masterQueue.getShardForQueue(queueId))
       }));
-      totalReclaimed += reclaimed;
+      if (this.concurrencyManager && reclaimedMessages.length > 0) {
+        try {
+          await this.concurrencyManager.releaseBatch(
+            reclaimedMessages.map((msg) => ({
+              queue: {
+                id: msg.queueId,
+                tenantId: msg.tenantId,
+                metadata: msg.metadata ?? {}
+              },
+              messageId: msg.messageId
+            }))
+          );
+        } catch (error) {
+          this.logger.error("Failed to release concurrency for reclaimed messages", {
+            count: reclaimedMessages.length,
+            error: error instanceof Error ? error.message : String(error)
+          });
+        }
+      }
+      totalReclaimed += reclaimedMessages.length;
     }
     if (totalReclaimed > 0) {
       this.logger.info("Reclaimed timed-out messages", { count: totalReclaimed });