@avada-falcon/worker-sdk 0.5.0 → 0.5.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@avada-falcon/worker-sdk",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Self-hosted background job runner powered by BullMQ + Redis",
   "type": "module",
   "main": "src/index.cjs",

package/src/worker/worker.js

@@ -12,10 +12,17 @@ import {FileLogger} from '../logging/fileLogger.js';
 import {LokiShipper} from '../logging/lokiShipper.js';
 import {Heartbeat} from './heartbeat.js';
 
-/** Key under job.data holding how many times the job has been declined by a worker. */
-const REQUEUE_COUNT_KEY = '__requeues';
-const DEFAULT_MAX_REQUEUES = 50;
+/** Keys stored under job.data to track admission back-off across requeues. */
+const REQUEUE_COUNT_KEY = '__requeues'; // how many times declined — for logging only
+const REQUEUE_FIRST_KEY = '__firstDeclinedAt'; // epoch ms of the FIRST decline — drives the cap
 const DEFAULT_REQUEUE_DELAY_MS = 200;
+// Runaway BACKSTOP for a stuck admission gate (e.g. a leaked memory budget that never
+// frees), NOT normal backpressure. Wall-clock based, not a requeue COUNT: a count cap of
+// 50 × ~200ms was only ~15s of tolerance — far less than a single job's timeout (up to
+// 540s) — so a job legitimately queued behind long-running jobs failed for merely waiting.
+// 30 min is comfortably longer than any single job, so only a truly stuck fleet trips it.
+// Override per job with jobConfig.maxRequeueWaitMs.
+const DEFAULT_MAX_REQUEUE_WAIT_MS = 30 * 60 * 1000;
 
 function requeueCount(job) {
   const count = job.data?.[REQUEUE_COUNT_KEY];
@@ -83,7 +90,7 @@ export function createWorker(configPath) {
       const handler = registry.get(job.name);
       const jobConfig = config.jobs[job.name] || {};
       const timeout = jobConfig.timeout || 30000;
-      const maxRequeues = jobConfig.maxRequeues ?? DEFAULT_MAX_REQUEUES;
+      const maxRequeueWaitMs = jobConfig.maxRequeueWaitMs ?? DEFAULT_MAX_REQUEUE_WAIT_MS;
 
       // Let a handler DECLINE to run on this worker and hand the job back to the queue,
       // delayed briefly, instead of blocking — a job blocked inside the handler stays
@@ -99,21 +106,31 @@ export function createWorker(configPath) {
       // then returns normally makes BullMQ complete a job that is also sitting in the
       // delayed set — it runs twice. Call it as `return ctx.requeue()` or let it throw.
       const requeue = async (retryMs = DEFAULT_REQUEUE_DELAY_MS) => {
+        const now = Date.now();
+        const data = job.data || {};
+        const firstDeclinedAt = data[REQUEUE_FIRST_KEY] || now;
+        const waitedMs = now - firstDeclinedAt;
         const count = requeueCount(job) + 1;
 
-        // No attempt is burned on a requeue, so an admission gate that never opens
-        // (leaked memory budget, gate bug) would ping-pong this job forever, silently.
-        // Cap it and fail loudly instead.
-        if (count > maxRequeues) {
+        // A requeue burns no attempt, so a gate that never opens (leaked memory budget,
+        // gate bug) would ping-pong this job forever, silently. Fail loudly — but only
+        // after a long WALL-CLOCK wait, so a job legitimately queued behind long-running
+        // jobs (up to their timeout) is never failed for merely waiting; only a truly
+        // stuck gate trips this.
+        if (waitedMs > maxRequeueWaitMs) {
           throw new Error(
-            `Job ${job.name}:${job.id} was declined ${count - 1} times ` +
-            `(maxRequeues: ${maxRequeues}) — admission gate never opened.`
+            `Job ${job.name}:${job.id} not admitted after ${Math.round(waitedMs / 1000)}s ` +
+            `over ${count} tries (maxRequeueWaitMs: ${maxRequeueWaitMs}ms) — admission gate stuck.`
           );
         }
 
-        // Persist the counter: the in-memory job.data copy is dropped once the job
-        // goes back to Redis, so it has to be written before re-scheduling.
-        await job.updateData({...job.data, [REQUEUE_COUNT_KEY]: count});
+        // Persist across the re-schedule: the in-memory job.data copy is dropped once the
+        // job goes back to Redis. Keep the FIRST-decline stamp stable; bump the info count.
+        await job.updateData({
+          ...data,
+          [REQUEUE_FIRST_KEY]: firstDeclinedAt,
+          [REQUEUE_COUNT_KEY]: count
+        });
 
         // Jitter the delay. Without it, every job this worker declines wakes at the
         // same instant and gets declined again in lockstep (thundering herd).
@@ -121,10 +138,12 @@ export function createWorker(configPath) {
 
         console.warn(
           `[worker-sdk] Declined ${job.name}:${job.id} — requeued in ${delay}ms ` +
-          `(${count}/${maxRequeues})`
+          `(try ${count}, waited ${Math.round(waitedMs / 1000)}s/${Math.round(
+            maxRequeueWaitMs / 1000
+          )}s)`
         );
 
-        await job.moveToDelayed(Date.now() + delay, token);
+        await job.moveToDelayed(now + delay, token);
         throw new DelayedError();
       };