@trigger.dev/redis-worker 0.0.0-prerelease-20251209154359 → 0.0.0-prerelease-20251211173228

package/dist/index.cjs

@@ -11784,12 +11784,7 @@ var ConcurrencyManager = class {
     );
     const keys = groupData.map((g) => g.key);
     const limits = groupData.map((g) => g.limit.toString());
-    const result = await this.redis.reserveConcurrency(
-      keys.length.toString(),
-      messageId,
-      ...keys,
-      ...limits
-    );
+    const result = await this.redis.reserveConcurrency(keys.length, keys, messageId, ...limits);
     return result === 1;
   }
   /**
@@ -11885,16 +11880,14 @@ var ConcurrencyManager = class {
   // ============================================================================
   #registerCommands() {
     this.redis.defineCommand("reserveConcurrency", {
-      numberOfKeys: 0,
-      // Will pass number of keys in ARGV
       lua: `
-local numGroups = tonumber(ARGV[1])
-local messageId = ARGV[2]
+local numGroups = #KEYS
+local messageId = ARGV[1]
 
 -- Check all groups first
 for i = 1, numGroups do
-  local key = ARGV[2 + i]  -- Keys start at ARGV[3]
-  local limit = tonumber(ARGV[2 + numGroups + i])  -- Limits come after keys
+  local key = KEYS[i]
+  local limit = tonumber(ARGV[1 + i])  -- Limits start at ARGV[2]
   local current = redis.call('SCARD', key)
   
   if current >= limit then
@@ -11904,7 +11897,7 @@ end
 
 -- All groups have capacity, add message to all
 for i = 1, numGroups do
-  local key = ARGV[2 + i]
+  local key = KEYS[i]
   redis.call('SADD', key, messageId)
 end
 
@@ -13222,7 +13215,20 @@ var DRRScheduler = class extends BaseScheduler {
     const eligibleTenants = tenantData.filter(
       (t) => !t.isAtCapacity && t.deficit >= 1
     );
+    const blockedTenants = tenantData.filter((t) => t.isAtCapacity);
+    if (blockedTenants.length > 0) {
+      this.logger.debug("DRR: tenants blocked by concurrency", {
+        blockedCount: blockedTenants.length,
+        blockedTenants: blockedTenants.map((t) => t.tenantId)
+      });
+    }
     eligibleTenants.sort((a, b) => b.deficit - a.deficit);
+    this.logger.debug("DRR: queue selection complete", {
+      totalQueues: queues.length,
+      totalTenants: tenantIds.length,
+      eligibleTenants: eligibleTenants.length,
+      topTenantDeficit: eligibleTenants[0]?.deficit
+    });
     return eligibleTenants.map((t) => ({
       tenantId: t.tenantId,
       queues: t.queues
@@ -13878,6 +13884,7 @@ var FairQueue = class {
     this.cooloffEnabled = options.cooloff?.enabled ?? true;
     this.cooloffThreshold = options.cooloff?.threshold ?? 10;
     this.cooloffPeriodMs = options.cooloff?.periodMs ?? 1e4;
+    this.globalRateLimiter = options.globalRateLimiter;
     this.telemetry = new FairQueueTelemetry({
       tracer: options.tracer,
       meter: options.meter,
@@ -13948,6 +13955,8 @@ var FairQueue = class {
   cooloffThreshold;
   cooloffPeriodMs;
   queueCooloffStates = /* @__PURE__ */ new Map();
+  // Global rate limiter
+  globalRateLimiter;
   // Runtime state
   messageHandler;
   isRunning = false;
@@ -13958,6 +13967,30 @@ var FairQueue = class {
   // Queue descriptor cache for message processing
   queueDescriptorCache = /* @__PURE__ */ new Map();
   // ============================================================================
+  // Public API - Telemetry
+  // ============================================================================
+  /**
+   * Register observable gauge callbacks for telemetry.
+   * Call this after FairQueue is created to enable gauge metrics.
+   *
+   * @param options.observedTenants - List of tenant IDs to observe for DLQ metrics
+   */
+  registerTelemetryGauges(options) {
+    this.telemetry.registerGaugeCallbacks({
+      getMasterQueueLength: async (shardId) => {
+        return await this.masterQueue.getShardQueueCount(shardId);
+      },
+      getInflightCount: async (shardId) => {
+        return await this.visibilityManager.getInflightCount(shardId);
+      },
+      getDLQLength: async (tenantId) => {
+        return await this.getDeadLetterQueueLength(tenantId);
+      },
+      shardCount: this.shardCount,
+      observedTenants: options?.observedTenants
+    });
+  }
+  // ============================================================================
   // Public API - Message Handler
   // ============================================================================
   /**
@@ -14409,6 +14442,16 @@ var FairQueue = class {
         return false;
       }
     }
+    if (this.globalRateLimiter) {
+      const result = await this.globalRateLimiter.limit();
+      if (!result.allowed && result.resetAt) {
+        const waitMs = Math.max(0, result.resetAt - Date.now());
+        if (waitMs > 0) {
+          this.logger.debug("Global rate limit reached, waiting", { waitMs, loopId });
+          await new Promise((resolve) => setTimeout(resolve, waitMs));
+        }
+      }
+    }
     const claimResult = await this.visibilityManager.claim(
       queueId,
       queueKey,
@@ -14534,18 +14577,33 @@ var FairQueue = class {
       return;
     }
     for (const { tenantId, queues } of tenantQueues) {
-      for (const queueId of queues) {
-        if (this.cooloffEnabled && this.#isInCooloff(queueId)) {
-          continue;
+      let availableSlots = 1;
+      if (this.concurrencyManager) {
+        const [current, limit] = await Promise.all([
+          this.concurrencyManager.getCurrentConcurrency("tenant", tenantId),
+          this.concurrencyManager.getConcurrencyLimit("tenant", tenantId)
+        ]);
+        availableSlots = Math.max(1, limit - current);
+      }
+      let slotsUsed = 0;
+      queueLoop: for (const queueId of queues) {
+        while (slotsUsed < availableSlots) {
+          if (this.cooloffEnabled && this.#isInCooloff(queueId)) {
+            break;
+          }
+          const processed = await this.#processOneMessage(loopId, queueId, tenantId, shardId);
+          if (processed) {
+            await this.scheduler.recordProcessed?.(tenantId, queueId);
+            this.#resetCooloff(queueId);
+            slotsUsed++;
+          } else {
+            this.#incrementCooloff(queueId);
+            break;
+          }
         }
-        const processed = await this.#processOneMessage(loopId, queueId, tenantId, shardId);
-        if (processed) {
-          await this.scheduler.recordProcessed?.(tenantId, queueId);
-          this.#resetCooloff(queueId);
-        } else {
-          this.#incrementCooloff(queueId);
+        if (slotsUsed >= availableSlots) {
+          break queueLoop;
         }
-        break;
       }
     }
   }
@@ -14564,6 +14622,16 @@ var FairQueue = class {
         return false;
       }
     }
+    if (this.globalRateLimiter) {
+      const result = await this.globalRateLimiter.limit();
+      if (!result.allowed && result.resetAt) {
+        const waitMs = Math.max(0, result.resetAt - Date.now());
+        if (waitMs > 0) {
+          this.logger.debug("Global rate limit reached, waiting", { waitMs, loopId });
+          await new Promise((resolve) => setTimeout(resolve, waitMs));
+        }
+      }
+    }
     const claimResult = await this.visibilityManager.claim(
       queueId,
       queueKey,
@@ -14888,6 +14956,11 @@ var FairQueue = class {
           tag: "cooloff",
           expiresAt: Date.now() + this.cooloffPeriodMs
         });
+        this.logger.debug("Queue entered cooloff", {
+          queueId,
+          cooloffPeriodMs: this.cooloffPeriodMs,
+          consecutiveFailures: newFailures
+        });
       } else {
         this.queueCooloffStates.set(queueId, {
           tag: "normal",