@trigger.dev/redis-worker 4.5.0-rc.6 → 4.5.0

package/dist/index.d.cts

@@ -64,6 +64,22 @@ declare class SimpleQueue<TMessageCatalog extends MessageCatalogSchema> {
     size({ includeFuture }?: {
         includeFuture?: boolean;
     }): Promise<number>;
+    /**
+     * Age (in ms) of the oldest *overdue* message — the oldest item whose scheduled
+     * time has already passed (score <= now). Returns 0 when the queue is empty or
+     * only holds future/delayed or in-flight (future-scored) items.
+     *
+     * Resolves the candidate against the `items` hash so orphaned `queue` entries
+     * (a member whose payload is missing — the same stale state `dequeueItems`
+     * cleans up) don't report a phantom stall for work that can't be dequeued. The
+     * Lua scans due items oldest-first and returns the first score whose payload
+     * still exists.
+     *
+     * This is the generic stall signal: it stays at 0 while a queue drains healthily
+     * and rises only when due work sits undrained (poison block, dead consumer,
+     * backpressure).
+     */
+    oldestMessageAge(): Promise<number>;
     getJob(id: string): Promise<QueueItem<TMessageCatalog> | null>;
     moveToDeadLetterQueue(id: string, errorMessage: string): Promise<void>;
     sizeOfDeadLetterQueue(): Promise<number>;
@@ -80,6 +96,7 @@ declare module "@internal/redis" {
         moveToDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, errorMessage: string, callback?: Callback<number>): Result<number, Context>;
         enqueueItemOnce(queue: string, items: string, id: string, score: number, serializedItem: string, callback?: Callback<number>): Result<number, Context>;
         getJob(queue: string, items: string, id: string, callback?: Callback<[string, string, string] | null>): Result<[string, string, string] | null, Context>;
+        getOldestDueScore(queue: string, items: string, now: number, callback?: Callback<string | number>): Result<string | number, Context>;
     }
 }
 

package/dist/index.d.ts

@@ -64,6 +64,22 @@ declare class SimpleQueue<TMessageCatalog extends MessageCatalogSchema> {
     size({ includeFuture }?: {
         includeFuture?: boolean;
     }): Promise<number>;
+    /**
+     * Age (in ms) of the oldest *overdue* message — the oldest item whose scheduled
+     * time has already passed (score <= now). Returns 0 when the queue is empty or
+     * only holds future/delayed or in-flight (future-scored) items.
+     *
+     * Resolves the candidate against the `items` hash so orphaned `queue` entries
+     * (a member whose payload is missing — the same stale state `dequeueItems`
+     * cleans up) don't report a phantom stall for work that can't be dequeued. The
+     * Lua scans due items oldest-first and returns the first score whose payload
+     * still exists.
+     *
+     * This is the generic stall signal: it stays at 0 while a queue drains healthily
+     * and rises only when due work sits undrained (poison block, dead consumer,
+     * backpressure).
+     */
+    oldestMessageAge(): Promise<number>;
     getJob(id: string): Promise<QueueItem<TMessageCatalog> | null>;
     moveToDeadLetterQueue(id: string, errorMessage: string): Promise<void>;
     sizeOfDeadLetterQueue(): Promise<number>;
@@ -80,6 +96,7 @@ declare module "@internal/redis" {
         moveToDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, errorMessage: string, callback?: Callback<number>): Result<number, Context>;
         enqueueItemOnce(queue: string, items: string, id: string, score: number, serializedItem: string, callback?: Callback<number>): Result<number, Context>;
         getJob(queue: string, items: string, id: string, callback?: Callback<[string, string, string] | null>): Result<[string, string, string] | null, Context>;
+        getOldestDueScore(queue: string, items: string, now: number, callback?: Callback<string | number>): Result<string | number, Context>;
     }
 }
 

package/dist/index.js

@@ -9985,6 +9985,37 @@ var SimpleQueue = class {
       throw e;
     }
   }
+  /**
+   * Age (in ms) of the oldest *overdue* message — the oldest item whose scheduled
+   * time has already passed (score <= now). Returns 0 when the queue is empty or
+   * only holds future/delayed or in-flight (future-scored) items.
+   *
+   * Resolves the candidate against the `items` hash so orphaned `queue` entries
+   * (a member whose payload is missing — the same stale state `dequeueItems`
+   * cleans up) don't report a phantom stall for work that can't be dequeued. The
+   * Lua scans due items oldest-first and returns the first score whose payload
+   * still exists.
+   *
+   * This is the generic stall signal: it stays at 0 while a queue drains healthily
+   * and rises only when due work sits undrained (poison block, dead consumer,
+   * backpressure).
+   */
+  async oldestMessageAge() {
+    try {
+      const now = Date.now();
+      const score = Number(await this.redis.getOldestDueScore(`queue`, `items`, now));
+      if (!Number.isFinite(score) || score < 0) {
+        return 0;
+      }
+      return Math.max(0, now - score);
+    } catch (e) {
+      this.logger.error(`SimpleQueue ${this.name}.oldestMessageAge(): error getting oldest age`, {
+        queue: this.name,
+        error: e
+      });
+      return 0;
+    }
+  }
   async getJob(id) {
     const result = await this.redis.getJob(`queue`, `items`, id);
     if (!result) {
@@ -10151,6 +10182,29 @@ var SimpleQueue = class {
         return dequeued
       `
     });
+    this.redis.defineCommand("getOldestDueScore", {
+      numberOfKeys: 2,
+      lua: `
+        local queue = KEYS[1]
+        local items = KEYS[2]
+        local now = tonumber(ARGV[1])
+
+        -- Oldest-first scan of due items, bounded so a long prefix of orphans can't
+        -- make this O(n). Orphans are rare (dequeueItems removes them), so in the
+        -- common case this returns on the first iteration. Read-only: unlike
+        -- dequeueItems we don't ZREM orphans here \u2014 a metric probe must not mutate.
+        local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, 100)
+
+        for i = 1, #result, 2 do
+          local id = result[i]
+          if redis.call('HEXISTS', items, id) == 1 then
+            return result[i + 1]
+          end
+        end
+
+        return -1
+      `
+    });
     this.redis.defineCommand("getJob", {
       numberOfKeys: 2,
       lua: `
@@ -11395,6 +11449,15 @@ var Worker = class _Worker {
     concurrencyLimitPendingObservableGauge.addCallback(
       this.#updateConcurrencyLimitPendingMetric.bind(this)
     );
+    const oldestMessageAgeObservableGauge = this.meter.createObservableGauge(
+      "redis_worker.queue.oldest_message_age",
+      {
+        description: "Age of the oldest overdue message in the queue",
+        unit: "ms",
+        valueType: ValueType.INT
+      }
+    );
+    oldestMessageAgeObservableGauge.addCallback(this.#updateOldestMessageAgeMetric.bind(this));
   }
   subscriber;
   tracer;
@@ -11423,6 +11486,12 @@ var Worker = class _Worker {
       worker_name: this.options.name
     });
   }
+  async #updateOldestMessageAgeMetric(observableResult) {
+    const oldestMessageAge = await this.queue.oldestMessageAge();
+    observableResult.observe(oldestMessageAge, {
+      worker_name: this.options.name
+    });
+  }
   async #updateConcurrencyLimitActiveMetric(observableResult) {
     for (const [workerId, limiter] of Object.entries(this.limiters)) {
       observableResult.observe(limiter.activeCount, {
@@ -11716,15 +11785,15 @@ var Worker = class _Worker {
               await this.flushBatch(queueItem.job, workerId, limiter);
             }
           } else {
-            limiter(
-              () => this.processItem(queueItem, items.length, workerId, limiter)
-            ).catch((err) => {
-              this.logger.error("Unhandled error in processItem:", {
-                error: err,
-                workerId,
-                item
-              });
-            });
+            limiter(() => this.processItem(queueItem, items.length, workerId, limiter)).catch(
+              (err) => {
+                this.logger.error("Unhandled error in processItem:", {
+                  error: err,
+                  workerId,
+                  item
+                });
+              }
+            );
           }
         }
       } catch (error) {
@@ -11874,13 +11943,25 @@ var Worker = class _Worker {
               attempt: newAttempt
             };
             if (!shouldLogError) {
-              this.logger.info(`Worker batch item reached max attempts. Moving to DLQ.`, dlqLogAttributes);
+              this.logger.info(
+                `Worker batch item reached max attempts. Moving to DLQ.`,
+                dlqLogAttributes
+              );
             } else if (errorLogLevel === "warn") {
-              this.logger.warn(`Worker batch item reached max attempts. Moving to DLQ.`, dlqLogAttributes);
+              this.logger.warn(
+                `Worker batch item reached max attempts. Moving to DLQ.`,
+                dlqLogAttributes
+              );
             } else if (errorLogLevel === "info") {
-              this.logger.info(`Worker batch item reached max attempts. Moving to DLQ.`, dlqLogAttributes);
+              this.logger.info(
+                `Worker batch item reached max attempts. Moving to DLQ.`,
+                dlqLogAttributes
+              );
             } else {
-              this.logger.error(`Worker batch item reached max attempts. Moving to DLQ.`, dlqLogAttributes);
+              this.logger.error(
+                `Worker batch item reached max attempts. Moving to DLQ.`,
+                dlqLogAttributes
+              );
             }
             await this.queue.moveToDeadLetterQueue(item.id, errorMessage);
             continue;
@@ -14389,9 +14470,7 @@ var DRRScheduler = class extends BaseScheduler {
         };
       })
     );
-    const eligibleTenants = tenantData.filter(
-      (t) => !t.isAtCapacity && t.deficit >= 1
-    );
+    const eligibleTenants = tenantData.filter((t) => !t.isAtCapacity && t.deficit >= 1);
     const blockedTenants = tenantData.filter((t) => t.isAtCapacity);
     if (blockedTenants.length > 0) {
       this.logger.debug("DRR: tenants blocked by concurrency", {
@@ -14684,11 +14763,7 @@ var WeightedScheduler = class extends BaseScheduler {
   // FairScheduler Implementation
   // ============================================================================
   async selectQueues(masterQueueShard, consumerId, context2) {
-    const snapshot = await this.#getOrCreateSnapshot(
-      masterQueueShard,
-      consumerId,
-      context2
-    );
+    const snapshot = await this.#getOrCreateSnapshot(masterQueueShard, consumerId, context2);
     if (snapshot.queues.length === 0) {
       return [];
     }
@@ -16061,10 +16136,7 @@ var FairQueue = class {
     if (this.concurrencyManager && storedMessage) {
       await this.concurrencyManager.release(descriptor, messageId);
     }
-    const { queueEmpty } = await this.#updateAllIndexesAfterDequeue(
-      queueId,
-      descriptor.tenantId
-    );
+    const { queueEmpty } = await this.#updateAllIndexesAfterDequeue(queueId, descriptor.tenantId);
     if (queueEmpty) {
       this.queueDescriptorCache.delete(queueId);
       this.queueCooloffStates.delete(queueId);
@@ -16321,38 +16393,6 @@ var FairQueue = class {
     }
     return false;
   }
-  #incrementCooloff(queueId) {
-    if (this.queueCooloffStates.size >= this.maxCooloffStatesSize) {
-      this.logger.warn("Cooloff states cache hit size cap, clearing all entries", {
-        size: this.queueCooloffStates.size,
-        cap: this.maxCooloffStatesSize
-      });
-      this.queueCooloffStates.clear();
-    }
-    const state = this.queueCooloffStates.get(queueId) ?? {
-      tag: "normal",
-      consecutiveFailures: 0
-    };
-    if (state.tag === "normal") {
-      const newFailures = state.consecutiveFailures + 1;
-      if (newFailures >= this.cooloffThreshold) {
-        this.queueCooloffStates.set(queueId, {
-          tag: "cooloff",
-          expiresAt: Date.now() + this.cooloffPeriodMs
-        });
-        this.logger.debug("Queue entered cooloff", {
-          queueId,
-          cooloffPeriodMs: this.cooloffPeriodMs,
-          consecutiveFailures: newFailures
-        });
-      } else {
-        this.queueCooloffStates.set(queueId, {
-          tag: "normal",
-          consecutiveFailures: newFailures
-        });
-      }
-    }
-  }
   #resetCooloff(queueId) {
     this.queueCooloffStates.delete(queueId);
   }
@@ -16620,7 +16660,10 @@ var stringToError = z.string().transform((v, ctx) => {
   try {
     return BufferEntryError.parse(JSON.parse(v));
   } catch {
-    ctx.addIssue({ code: z.ZodIssueCode.custom, message: "expected JSON-encoded BufferEntryError" });
+    ctx.addIssue({
+      code: z.ZodIssueCode.custom,
+      message: "expected JSON-encoded BufferEntryError"
+    });
     return z.NEVER;
   }
 });
@@ -16818,11 +16861,7 @@ var MollifierBuffer = class {
   // not an error.
   async listEntriesForEnv(envId, maxCount) {
     if (maxCount <= 0) return [];
-    const runIds = await this.redis.lrange(
-      `mollifier:queue:${envId}`,
-      0,
-      maxCount - 1
-    );
+    const runIds = await this.redis.lrange(`mollifier:queue:${envId}`, 0, maxCount - 1);
     if (runIds.length === 0) return [];
     const pipeline = this.redis.pipeline();
     for (const runId of runIds) {
@@ -16959,10 +16998,7 @@ var MollifierBuffer = class {
   // never wiped by a slow predecessor.
   async releaseClaim(input) {
     const claimKey = makeIdempotencyClaimKey(input);
-    await this.redis.releaseMollifierClaim(
-      claimKey,
-      `${PENDING_PREFIX}${input.token}`
-    );
+    await this.redis.releaseMollifierClaim(claimKey, `${PENDING_PREFIX}${input.token}`);
   }
   // Read the current claim value, used by the wait/poll loop on losers
   // to detect "pending" → "resolved" transitions and timeouts.
@@ -17647,9 +17683,7 @@ var MollifierDrainer = class {
     const orgs = await this.buffer.listOrgs();
     if (orgs.length === 0) return { drained: 0, failed: 0 };
     const orgSlice = this.takeOrgSlice(orgs);
-    const envsByOrg = await Promise.all(
-      orgSlice.map((orgId) => this.buffer.listEnvsForOrg(orgId))
-    );
+    const envsByOrg = await Promise.all(orgSlice.map((orgId) => this.buffer.listEnvsForOrg(orgId)));
     const targets = [];
     for (let i = 0; i < orgSlice.length; i++) {
       const orgId = orgSlice[i];
@@ -17862,14 +17896,11 @@ var MollifierDrainer = class {
         } catch (writeErr) {
           if (this.isRetryable(writeErr)) {
             await this.buffer.requeue(entry.runId);
-            this.logger.warn(
-              "MollifierDrainer: terminal-failure callback retryable; requeued",
-              {
-                runId: entry.runId,
-                attempts: nextAttempts,
-                writeErr
-              }
-            );
+            this.logger.warn("MollifierDrainer: terminal-failure callback retryable; requeued", {
+              runId: entry.runId,
+              attempts: nextAttempts,
+              writeErr
+            });
             return "failed";
           }
           this.logger.error("MollifierDrainer: terminal-failure callback failed", {