@trigger.dev/redis-worker 0.0.0-prerelease-20251209163704 → 0.0.0-prerelease-20251215135620

package/dist/index.cjs

@@ -11784,12 +11784,7 @@ var ConcurrencyManager = class {
     );
     const keys = groupData.map((g) => g.key);
     const limits = groupData.map((g) => g.limit.toString());
-    const result = await this.redis.reserveConcurrency(
-      keys.length.toString(),
-      messageId,
-      ...keys,
-      ...limits
-    );
+    const result = await this.redis.reserveConcurrency(keys.length, keys, messageId, ...limits);
     return result === 1;
   }
   /**
@@ -11885,16 +11880,14 @@ var ConcurrencyManager = class {
   // ============================================================================
   #registerCommands() {
     this.redis.defineCommand("reserveConcurrency", {
-      numberOfKeys: 0,
-      // Will pass number of keys in ARGV
       lua: `
-local numGroups = tonumber(ARGV[1])
-local messageId = ARGV[2]
+local numGroups = #KEYS
+local messageId = ARGV[1]
 
 -- Check all groups first
 for i = 1, numGroups do
-  local key = ARGV[2 + i]  -- Keys start at ARGV[3]
-  local limit = tonumber(ARGV[2 + numGroups + i])  -- Limits come after keys
+  local key = KEYS[i]
+  local limit = tonumber(ARGV[1 + i])  -- Limits start at ARGV[2]
   local current = redis.call('SCARD', key)
   
   if current >= limit then
@@ -11904,7 +11897,7 @@ end
 
 -- All groups have capacity, add message to all
 for i = 1, numGroups do
-  local key = ARGV[2 + i]
+  local key = KEYS[i]
   redis.call('SADD', key, messageId)
 end
 
@@ -12517,8 +12510,12 @@ var VisibilityManager = class {
     const inflightKey = this.keys.inflightKey(shardId);
     const member = this.#makeMember(messageId, queueId);
     const newDeadline = Date.now() + extendMs;
-    const result = await this.redis.zadd(inflightKey, "XX", newDeadline, member);
-    const success = result !== 0;
+    const result = await this.redis.heartbeatMessage(
+      inflightKey,
+      member,
+      newDeadline.toString()
+    );
+    const success = result === 1;
     if (success) {
       this.logger.debug("Heartbeat successful", {
         messageId,
@@ -12775,6 +12772,24 @@ redis.call('HDEL', inflightDataKey, messageId)
 redis.call('ZADD', queueKey, score, messageId)
 redis.call('HSET', queueItemsKey, messageId, payload)
 
+return 1
+      `
+    });
+    this.redis.defineCommand("heartbeatMessage", {
+      numberOfKeys: 1,
+      lua: `
+local inflightKey = KEYS[1]
+local member = ARGV[1]
+local newDeadline = tonumber(ARGV[2])
+
+-- Check if member exists in the in-flight set
+local score = redis.call('ZSCORE', inflightKey, member)
+if not score then
+  return 0
+end
+
+-- Update the deadline
+redis.call('ZADD', inflightKey, 'XX', newDeadline, member)
 return 1
       `
     });
@@ -13222,7 +13237,20 @@ var DRRScheduler = class extends BaseScheduler {
     const eligibleTenants = tenantData.filter(
       (t) => !t.isAtCapacity && t.deficit >= 1
     );
+    const blockedTenants = tenantData.filter((t) => t.isAtCapacity);
+    if (blockedTenants.length > 0) {
+      this.logger.debug("DRR: tenants blocked by concurrency", {
+        blockedCount: blockedTenants.length,
+        blockedTenants: blockedTenants.map((t) => t.tenantId)
+      });
+    }
     eligibleTenants.sort((a, b) => b.deficit - a.deficit);
+    this.logger.debug("DRR: queue selection complete", {
+      totalQueues: queues.length,
+      totalTenants: tenantIds.length,
+      eligibleTenants: eligibleTenants.length,
+      topTenantDeficit: eligibleTenants[0]?.deficit
+    });
     return eligibleTenants.map((t) => ({
       tenantId: t.tenantId,
       queues: t.queues
@@ -13527,7 +13555,10 @@ var WeightedScheduler = class extends BaseScheduler {
       return { tenantId, avgAge };
     });
     const maxAge = Math.max(...tenantAges.map((t) => t.avgAge));
-    const weightedTenants = tenantAges.map((t) => ({
+    const weightedTenants = maxAge === 0 ? tenantAges.map((t) => ({
+      tenantId: t.tenantId,
+      weight: 1 / tenantAges.length
+    })) : tenantAges.map((t) => ({
       tenantId: t.tenantId,
       weight: t.avgAge / maxAge
     }));
@@ -13570,11 +13601,11 @@ var WeightedScheduler = class extends BaseScheduler {
       const tenant = snapshot.tenants.get(tenantId);
       let weight = 1;
       if (concurrencyLimitBias > 0) {
-        const normalizedLimit = tenant.concurrency.limit / maxLimit;
+        const normalizedLimit = maxLimit > 0 ? tenant.concurrency.limit / maxLimit : 0;
         weight *= 1 + Math.pow(normalizedLimit * concurrencyLimitBias, 2);
       }
       if (availableCapacityBias > 0) {
-        const usedPercentage = tenant.concurrency.current / tenant.concurrency.limit;
+        const usedPercentage = tenant.concurrency.limit > 0 ? tenant.concurrency.current / tenant.concurrency.limit : 1;
         const availableBonus = 1 - usedPercentage;
         weight *= 1 + Math.pow(availableBonus * availableCapacityBias, 2);
       }
@@ -13593,9 +13624,10 @@ var WeightedScheduler = class extends BaseScheduler {
       return queues.sort((a, b) => b.age - a.age).map((q) => q.queueId);
     }
     const maxAge = Math.max(...queues.map((q) => q.age));
+    const ageDenom = maxAge === 0 ? 1 : maxAge;
     const weightedQueues = queues.map((q) => ({
       queue: q,
-      weight: 1 + q.age / maxAge * queueAgeRandomization
+      weight: 1 + q.age / ageDenom * queueAgeRandomization
     }));
     const result = [];
     let remaining = [...weightedQueues];
@@ -13878,6 +13910,7 @@ var FairQueue = class {
     this.cooloffEnabled = options.cooloff?.enabled ?? true;
     this.cooloffThreshold = options.cooloff?.threshold ?? 10;
     this.cooloffPeriodMs = options.cooloff?.periodMs ?? 1e4;
+    this.globalRateLimiter = options.globalRateLimiter;
     this.telemetry = new FairQueueTelemetry({
       tracer: options.tracer,
       meter: options.meter,
@@ -13948,6 +13981,8 @@ var FairQueue = class {
   cooloffThreshold;
   cooloffPeriodMs;
   queueCooloffStates = /* @__PURE__ */ new Map();
+  // Global rate limiter
+  globalRateLimiter;
   // Runtime state
   messageHandler;
   isRunning = false;
@@ -13958,6 +13993,30 @@ var FairQueue = class {
   // Queue descriptor cache for message processing
   queueDescriptorCache = /* @__PURE__ */ new Map();
   // ============================================================================
+  // Public API - Telemetry
+  // ============================================================================
+  /**
+   * Register observable gauge callbacks for telemetry.
+   * Call this after FairQueue is created to enable gauge metrics.
+   *
+   * @param options.observedTenants - List of tenant IDs to observe for DLQ metrics
+   */
+  registerTelemetryGauges(options) {
+    this.telemetry.registerGaugeCallbacks({
+      getMasterQueueLength: async (shardId) => {
+        return await this.masterQueue.getShardQueueCount(shardId);
+      },
+      getInflightCount: async (shardId) => {
+        return await this.visibilityManager.getInflightCount(shardId);
+      },
+      getDLQLength: async (tenantId) => {
+        return await this.getDeadLetterQueueLength(tenantId);
+      },
+      shardCount: this.shardCount,
+      observedTenants: options?.observedTenants
+    });
+  }
+  // ============================================================================
   // Public API - Message Handler
   // ============================================================================
   /**
@@ -14409,6 +14468,16 @@ var FairQueue = class {
         return false;
       }
     }
+    if (this.globalRateLimiter) {
+      const result = await this.globalRateLimiter.limit();
+      if (!result.allowed && result.resetAt) {
+        const waitMs = Math.max(0, result.resetAt - Date.now());
+        if (waitMs > 0) {
+          this.logger.debug("Global rate limit reached, waiting", { waitMs, loopId });
+          await new Promise((resolve) => setTimeout(resolve, waitMs));
+        }
+      }
+    }
     const claimResult = await this.visibilityManager.claim(
       queueId,
       queueKey,
@@ -14534,18 +14603,33 @@ var FairQueue = class {
       return;
     }
     for (const { tenantId, queues } of tenantQueues) {
-      for (const queueId of queues) {
-        if (this.cooloffEnabled && this.#isInCooloff(queueId)) {
-          continue;
+      let availableSlots = 1;
+      if (this.concurrencyManager) {
+        const [current, limit] = await Promise.all([
+          this.concurrencyManager.getCurrentConcurrency("tenant", tenantId),
+          this.concurrencyManager.getConcurrencyLimit("tenant", tenantId)
+        ]);
+        availableSlots = Math.max(1, limit - current);
+      }
+      let slotsUsed = 0;
+      queueLoop: for (const queueId of queues) {
+        while (slotsUsed < availableSlots) {
+          if (this.cooloffEnabled && this.#isInCooloff(queueId)) {
+            break;
+          }
+          const processed = await this.#processOneMessage(loopId, queueId, tenantId, shardId);
+          if (processed) {
+            await this.scheduler.recordProcessed?.(tenantId, queueId);
+            this.#resetCooloff(queueId);
+            slotsUsed++;
+          } else {
+            this.#incrementCooloff(queueId);
+            break;
+          }
         }
-        const processed = await this.#processOneMessage(loopId, queueId, tenantId, shardId);
-        if (processed) {
-          await this.scheduler.recordProcessed?.(tenantId, queueId);
-          this.#resetCooloff(queueId);
-        } else {
-          this.#incrementCooloff(queueId);
+        if (slotsUsed >= availableSlots) {
+          break queueLoop;
         }
-        break;
       }
     }
   }
@@ -14564,6 +14648,16 @@ var FairQueue = class {
         return false;
       }
     }
+    if (this.globalRateLimiter) {
+      const result = await this.globalRateLimiter.limit();
+      if (!result.allowed && result.resetAt) {
+        const waitMs = Math.max(0, result.resetAt - Date.now());
+        if (waitMs > 0) {
+          this.logger.debug("Global rate limit reached, waiting", { waitMs, loopId });
+          await new Promise((resolve) => setTimeout(resolve, waitMs));
+        }
+      }
+    }
     const claimResult = await this.visibilityManager.claim(
       queueId,
       queueKey,
@@ -14610,6 +14704,17 @@ var FairQueue = class {
           error: result.error.message
         });
         await this.#moveToDeadLetterQueue(storedMessage, "Payload validation failed");
+        if (this.concurrencyManager) {
+          try {
+            await this.concurrencyManager.release(descriptor, storedMessage.id);
+          } catch (releaseError) {
+            this.logger.error("Failed to release concurrency slot after payload validation failure", {
+              messageId: storedMessage.id,
+              queueId,
+              error: releaseError instanceof Error ? releaseError.message : String(releaseError)
+            });
+          }
+        }
         return;
       }
       payload = result.data;
@@ -14786,7 +14891,6 @@ var FairQueue = class {
   }
   async #moveToDeadLetterQueue(storedMessage, errorMessage) {
     if (!this.deadLetterQueueEnabled) {
-      this.masterQueue.getShardForQueue(storedMessage.queueId);
       await this.visibilityManager.complete(storedMessage.id, storedMessage.queueId);
       return;
     }
@@ -14888,6 +14992,11 @@ var FairQueue = class {
           tag: "cooloff",
           expiresAt: Date.now() + this.cooloffPeriodMs
         });
+        this.logger.debug("Queue entered cooloff", {
+          queueId,
+          cooloffPeriodMs: this.cooloffPeriodMs,
+          consecutiveFailures: newFailures
+        });
       } else {
         this.queueCooloffStates.set(queueId, {
           tag: "normal",