@trigger.dev/redis-worker 0.0.0-prerelease-20250703115146 → 0.0.0-prerelease-20250715205134

package/dist/index.cjs

@@ -11084,7 +11084,6 @@ var Worker = class _Worker {
     this.jobs = options.jobs;
     const { workers = 1, tasksPerWorker = 1, limit = 10 } = options.concurrency ?? {};
     this.concurrency = { workers, tasksPerWorker, limit };
-    this.limiter = pLimit(this.concurrency.limit);
     const masterQueueObservableGauge = this.meter.createObservableGauge("redis_worker.queue.size", {
       description: "The number of items in the queue",
       unit: "items",
@@ -11135,7 +11134,7 @@ var Worker = class _Worker {
   concurrency;
   shutdownTimeoutMs;
   // The p-limit limiter to control overall concurrency.
-  limiter;
+  limiters = {};
   async #updateQueueSizeMetric(observableResult) {
     const queueSize = await this.queue.size();
     observableResult.observe(queueSize, {
@@ -11149,14 +11148,20 @@ var Worker = class _Worker {
     });
   }
   async #updateConcurrencyLimitActiveMetric(observableResult) {
-    observableResult.observe(this.limiter.activeCount, {
-      worker_name: this.options.name
-    });
+    for (const [workerId, limiter] of Object.entries(this.limiters)) {
+      observableResult.observe(limiter.activeCount, {
+        worker_name: this.options.name,
+        worker_id: workerId
+      });
+    }
   }
   async #updateConcurrencyLimitPendingMetric(observableResult) {
-    observableResult.observe(this.limiter.pendingCount, {
-      worker_name: this.options.name
-    });
+    for (const [workerId, limiter] of Object.entries(this.limiters)) {
+      observableResult.observe(limiter.pendingCount, {
+        worker_name: this.options.name,
+        worker_id: workerId
+      });
+    }
   }
   start() {
     const { workers, tasksPerWorker } = this.concurrency;
@@ -11325,6 +11330,8 @@ var Worker = class _Worker {
    * processes them, and then waits before the next iteration.
    */
   async runWorkerLoop(workerId, taskCount, workerIndex, totalWorkers) {
+    const limiter = pLimit(this.concurrency.limit);
+    this.limiters[workerId] = limiter;
     const pollIntervalMs = this.options.pollIntervalMs ?? 1e3;
     const immediatePollIntervalMs = this.options.immediatePollIntervalMs ?? 100;
     const delayBetweenWorkers = this.options.pollIntervalMs ?? 1e3;
@@ -11341,31 +11348,35 @@ var Worker = class _Worker {
       concurrencyOptions: this.concurrency
     });
     while (!this.isShuttingDown) {
-      if (this.limiter.activeCount + this.limiter.pendingCount >= this.concurrency.limit) {
+      if (limiter.activeCount + limiter.pendingCount >= this.concurrency.limit) {
         this.logger.debug("Worker at capacity, waiting", {
           workerId,
           concurrencyOptions: this.concurrency,
-          activeCount: this.limiter.activeCount,
-          pendingCount: this.limiter.pendingCount
+          activeCount: limiter.activeCount,
+          pendingCount: limiter.pendingCount
         });
         await _Worker.delay(pollIntervalMs);
         continue;
       }
+      const $taskCount = Math.min(
+        taskCount,
+        this.concurrency.limit - limiter.activeCount - limiter.pendingCount
+      );
       try {
         const items = await this.withHistogram(
           this.metrics.dequeueDuration,
-          this.queue.dequeue(taskCount),
+          this.queue.dequeue($taskCount),
           {
             worker_id: workerId,
-            task_count: taskCount
+            task_count: $taskCount
           }
         );
         if (items.length === 0) {
           this.logger.debug("No items to dequeue", {
             workerId,
             concurrencyOptions: this.concurrency,
-            activeCount: this.limiter.activeCount,
-            pendingCount: this.limiter.pendingCount
+            activeCount: limiter.activeCount,
+            pendingCount: limiter.pendingCount
           });
           await _Worker.delay(pollIntervalMs);
           continue;
@@ -11374,15 +11385,15 @@ var Worker = class _Worker {
           workerId,
           itemCount: items.length,
           concurrencyOptions: this.concurrency,
-          activeCount: this.limiter.activeCount,
-          pendingCount: this.limiter.pendingCount
+          activeCount: limiter.activeCount,
+          pendingCount: limiter.pendingCount
         });
         for (const item of items) {
-          this.limiter(() => this.processItem(item, items.length, workerId)).catch(
-            (err) => {
-              this.logger.error("Unhandled error in processItem:", { error: err, workerId, item });
-            }
-          );
+          limiter(
+            () => this.processItem(item, items.length, workerId, limiter)
+          ).catch((err) => {
+            this.logger.error("Unhandled error in processItem:", { error: err, workerId, item });
+          });
         }
       } catch (error) {
         this.logger.error("Error dequeuing items:", { name: this.options.name, error });
@@ -11396,7 +11407,7 @@ var Worker = class _Worker {
   /**
    * Processes a single item.
    */
-  async processItem({ id, job, item, visibilityTimeoutMs, attempt, timestamp, deduplicationKey }, batchSize, workerId) {
+  async processItem({ id, job, item, visibilityTimeoutMs, attempt, timestamp, deduplicationKey }, batchSize, workerId, limiter) {
     const catalogItem = this.options.catalog[job];
     const handler = this.jobs[job];
     if (!handler) {
@@ -11435,9 +11446,9 @@ var Worker = class _Worker {
           job_timestamp: timestamp.getTime(),
           job_age_in_ms: Date.now() - timestamp.getTime(),
           worker_id: workerId,
-          worker_limit_concurrency: this.limiter.concurrency,
-          worker_limit_active: this.limiter.activeCount,
-          worker_limit_pending: this.limiter.pendingCount,
+          worker_limit_concurrency: limiter.concurrency,
+          worker_limit_active: limiter.activeCount,
+          worker_limit_pending: limiter.pendingCount,
           worker_name: this.options.name,
           batch_size: batchSize
         }