@blamejs/core 0.18.42 → 0.18.44

package/lib/queue.js

@@ -136,6 +136,7 @@ var sweepTimer = null;
  *     },
  *   },
  *   defaultBackend?: string,  // name to use when enqueue/consume omit { backend }
+ *   sweepIntervalMs?: number, // expired-lease sweep period; default 30s, 0 = no timer
  *
  * @example
  *   b.queue.init({
@@ -153,6 +154,20 @@ function init(opts) {
   if (!opts || !opts.backends) {
     throw _err("INVALID_CONFIG", "queue.init({ backends }) is required", true);
   }
+  // Validate every option BEFORE touching module state. `initialized` stays
+  // false on a throw, so a rejected init is expected to leave nothing behind —
+  // but `backends` and `defaultBackend` are module-level and a later init does
+  // not clear them, so validating after assignment would leave the refused
+  // configuration's queues visible through listBackends() and selectable by
+  // name.
+  var sweepIntervalMs = opts.sweepIntervalMs === undefined
+    ? C.TIME.seconds(30)
+    : opts.sweepIntervalMs;
+  if (sweepIntervalMs !== 0 && !numericChecks.isPositiveInt(sweepIntervalMs)) {
+    throw _err("INVALID_CONFIG",
+      "queue.init({ sweepIntervalMs }) must be a positive integer, or 0 to run no " +
+      "sweep timer; got " + JSON.stringify(opts.sweepIntervalMs), true);
+  }
 
   // Self-register the _blamejs_jobs sealed-column declaration so payload +
   // lastError seal at rest even when this process never ran db.init (a
@@ -218,13 +233,21 @@ function init(opts) {
 
   // Sweep expired leases periodically (every 30s) so crashed-handler jobs
   // get re-pended.
-  sweepTimer = safeAsync.repeating(function () {
+  //
+  // `sweepIntervalMs: 0` starts no timer at all. That is the mode a scheduled
+  // runtime wants — a Cloudflare `scheduled()` handler, a Lambda, a CronJob —
+  // where a resident timer is not merely useless but wrong: the isolate is
+  // frozen between invocations, so it either never fires, or fires inside a
+  // LATER invocation and touches the backend outside any request the platform
+  // is accounting for. b.queue.tick sweeps once per invocation itself, so
+  // nothing is lost by turning the timer off there.
+  sweepTimer = sweepIntervalMs === 0 ? null : safeAsync.repeating(function () {
     Object.keys(backends).forEach(function (n) {
       if (backends[n].sweepExpired) {
         backends[n].sweepExpired().catch(function () { /* best effort */ });
       }
     });
-  }, C.TIME.seconds(30), { name: "queue-sweep" });
+  }, sweepIntervalMs, { name: "queue-sweep" });
 
   initialized = true;
 }
@@ -394,10 +417,6 @@ function consume(queueName, handler, opts) {
     if (rateLimit) rateLimit.timestamps.push(Date.now());
   }
 
-  // Progress audit-emit rate-limit — protect the audit chain from a
-  // chatty handler that calls progress() every loop iteration.
-  var PROGRESS_MIN_INTERVAL_MS = 250;
-
   // Each consumer has its own AbortController so cancel() unblocks any
   // in-flight poll-sleep immediately rather than waiting up to
   // pollIntervalMs (default 1s) for the next while-loop iteration.
@@ -463,101 +482,13 @@ function consume(queueName, handler, opts) {
           _emit("system.queue.consume.start", {
             metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
           });
-          // Consume a rate-limit slot at handler-start so the budget
-          // tracks invocation rate, not lease rate (a single lease that
-          // splits work across many sub-units doesn't double-count).
+          // Consume a rate-limit slot at handler-start so the budget tracks
+          // invocation rate, not lease rate (a single lease that splits work
+          // across many sub-units doesn't double-count).
           _rateLimitConsume();
-
-          // Handler context — second arg to handler. Carries
-          // ctx.extendLease(ms) for long-running handlers and
-          // ctx.progress(0..100) for surfacing job progress to the
-          // audit chain (rate-limited so chatty handlers don't drown it).
-          var lastProgressEmitAt = 0;
-          var lastProgressValue = -1;
-          var ctx = {
-            extendLease: function (additionalMs) {
-              if (typeof b.extendLease !== "function") {
-                throw _err("EXTEND_LEASE_UNSUPPORTED",
-                  "queue backend '" + b.name + "' does not support extendLease",
-                  true);
-              }
-              return b.extendLease(job.jobId, additionalMs, { attempt: job.attempts }).then(function (ok) {
-                if (ok) {
-                  _emit("system.queue.lease.extended", {
-                    metadata: { queue: queueName, backend: b.name, jobId: job.jobId, additionalMs: additionalMs },
-                  });
-                }
-                return ok;
-              });
-            },
-            progress: function (pct) {
-              if (typeof pct !== "number" || !isFinite(pct)) return;
-              var clamped = Math.max(0, Math.min(100, Math.floor(pct)));
-              var now = Date.now();
-              // Always emit 0 and 100 (start/done markers); throttle the rest.
-              var isMarker = clamped === 0 || clamped === 100;
-              if (!isMarker && (now - lastProgressEmitAt) < PROGRESS_MIN_INTERVAL_MS) return;
-              if (clamped === lastProgressValue && !isMarker) return;
-              lastProgressEmitAt = now;
-              lastProgressValue = clamped;
-              observability.event("queue.progress", clamped, { queueName: queueName });
-              _emit("system.queue.progress", {
-                metadata: {
-                  queue: queueName, backend: b.name, jobId: job.jobId,
-                  attempt: job.attempts, traceId: job.traceId,
-                  percent: clamped,
-                },
-              });
-            },
-          };
-          observability.tap("queue.consume",
-            { queueName: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts },
-            function () {
-              return Promise.resolve()
-                .then(function () { return handler(job, ctx); })
-                .then(function () {
-                  return b.complete(job.jobId, { attempt: job.attempts }).then(function () {
-                    _emit("system.queue.consume.success", {
-                      metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
-                    });
-                    observability.event("queue.complete", 1, { queueName: queueName });
-                  });
-                }, function (err) {
-              var msg = (err && err.message) || String(err);
-              var willRetry = job.attempts < job.maxAttempts;
-              return b.fail(job.jobId, msg, { retryDelayMs: _backoffDelay(job.attempts), attempt: job.attempts })
-                .then(function () {
-                  observability.event("queue.fail", 1, { queueName: queueName, willRetry: willRetry });
-                  _emit("system.queue.consume.failure", {
-                    metadata: {
-                      queue:    queueName, backend: b.name, jobId: job.jobId,
-                      attempt:  job.attempts, traceId: job.traceId,
-                      maxAttempts: job.maxAttempts, willRetry: willRetry,
-                    },
-                    reason:   msg,
-                    outcome:  "failure",
-                  });
-                  // DLQ-write event when the job has exhausted its retries.
-                  // Operators wire this to their alerting / dashboards
-                  // — failed-after-retries is "needs human review" not
-                  // "in the normal flow." Audit chain captures the
-                  // final state for forensics.
-                  if (!willRetry) {
-                    _emit("system.queue.dlq.write", {
-                      metadata: {
-                        queue: queueName, backend: b.name, jobId: job.jobId,
-                        attempts: job.attempts, traceId: job.traceId,
-                      },
-                      reason:  msg,
-                      outcome: "failure",
-                    });
-                  }
-                });
-            })
+          _runJob(b, queueName, job, handler)
             .catch(function (_e) { /* lifecycle errors swallowed — operator sees via audit */ })
             .then(function () { state.inFlight.delete(job.jobId); });
-            }
-          );
         })(jobs[i]);
       }
       await _pollSleep(fastPollMs);
@@ -576,6 +507,318 @@ function _backoffDelay(attempt) {
   return retryHelper.backoffDelay(attempt, _QUEUE_BACKOFF_OPTS);
 }
 
+// The handler context passed as the second argument to a job handler:
+// ctx.extendLease(ms) for a handler that outlives its lease, ctx.progress(0..100)
+// for surfacing progress to the audit chain (rate-limited so a chatty handler
+// cannot drown it).
+var _PROGRESS_MIN_INTERVAL_MS = 250;
+function _handlerContext(backend, queueName, job) {
+  var lastProgressEmitAt = 0;
+  var lastProgressValue = -1;
+  return {
+    extendLease: function (additionalMs) {
+      if (typeof backend.extendLease !== "function") {
+        throw _err("EXTEND_LEASE_UNSUPPORTED",
+          "queue backend '" + backend.name + "' does not support extendLease", true);
+      }
+      return backend.extendLease(job.jobId, additionalMs, { attempt: job.attempts }).then(function (ok) {
+        if (ok) {
+          _emit("system.queue.lease.extended", {
+            metadata: { queue: queueName, backend: backend.name, jobId: job.jobId, additionalMs: additionalMs },
+          });
+        }
+        return ok;
+      });
+    },
+    progress: function (pct) {
+      if (typeof pct !== "number" || !isFinite(pct)) return;
+      var clamped = Math.max(0, Math.min(100, Math.floor(pct)));
+      var now = Date.now();
+      // Always emit 0 and 100 (start/done markers); throttle the rest.
+      var isMarker = clamped === 0 || clamped === 100;
+      if (!isMarker && (now - lastProgressEmitAt) < _PROGRESS_MIN_INTERVAL_MS) return;
+      if (clamped === lastProgressValue && !isMarker) return;
+      lastProgressEmitAt = now;
+      lastProgressValue = clamped;
+      observability.event("queue.progress", clamped, { queueName: queueName });
+      _emit("system.queue.progress", {
+        metadata: {
+          queue: queueName, backend: backend.name, jobId: job.jobId,
+          attempt: job.attempts, traceId: job.traceId, percent: clamped,
+        },
+      });
+    },
+  };
+}
+
+// Run one leased job to its terminal state: complete on success, or fail with
+// the deterministic backoff and — once the attempts are exhausted — the DLQ
+// event. Resolves to "succeeded" or "failed"; `movedToDlq` says whether this
+// failure was the last one.
+//
+// Both drivers go through here. b.queue.consume runs it inside a resident
+// polling loop; b.queue.tick runs it over one leased batch and returns. The
+// retry schedule, the attempt accounting and the DLQ transition are the part
+// that is genuinely hard to get right, so there is exactly one copy of it.
+function _runJob(backend, queueName, job, handler) {
+  var ctx = _handlerContext(backend, queueName, job);
+  return observability.tap("queue.consume",
+    { queueName: queueName, backend: backend.name, jobId: job.jobId, attempt: job.attempts },
+    function () {
+      return Promise.resolve()
+        .then(function () { return handler(job, ctx); })
+        .then(function () {
+          return backend.complete(job.jobId, { attempt: job.attempts }).then(function (settled) {
+            // complete() answers FALSE when the attempt fence no longer
+            // matches: this worker's lease expired, the sweep re-pended the
+            // job, and someone else owns it now. Nothing was settled here, so
+            // claiming success would emit a success event for work this worker
+            // did not finish and inflate the caller's tally. Report it as
+            // stale and stay quiet.
+            if (settled === false) {
+              observability.event("queue.stale", 1, { queueName: queueName });
+              return { status: "stale", movedToDlq: false };
+            }
+            _emit("system.queue.consume.success", {
+              metadata: { queue: queueName, backend: backend.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
+            });
+            observability.event("queue.complete", 1, { queueName: queueName });
+            // `status`, not `outcome` — `outcome` is the audit-event
+            // vocabulary (success / failure / denied) and this is the job's
+            // terminal state for the caller, not an audit row.
+            return { status: "succeeded", movedToDlq: false };
+          });
+        }, function (err) {
+          var msg = (err && err.message) || String(err);
+          var willRetry = job.attempts < job.maxAttempts;
+          return backend.fail(job.jobId, msg, { retryDelayMs: _backoffDelay(job.attempts), attempt: job.attempts })
+            .then(function (settled) {
+              // Same fence as complete(): a false answer means the job was
+              // re-leased under this worker, so this failure did not land.
+              // Emitting the failure — or worse, the DLQ write — would report
+              // a terminal state for a job that is running elsewhere.
+              if (settled === false) {
+                observability.event("queue.stale", 1, { queueName: queueName });
+                return { status: "stale", movedToDlq: false };
+              }
+              observability.event("queue.fail", 1, { queueName: queueName, willRetry: willRetry });
+              _emit("system.queue.consume.failure", {
+                metadata: {
+                  queue:    queueName, backend: backend.name, jobId: job.jobId,
+                  attempt:  job.attempts, traceId: job.traceId,
+                  maxAttempts: job.maxAttempts, willRetry: willRetry,
+                },
+                reason:   msg,
+                outcome:  "failure",
+              });
+              // DLQ-write event when the job has exhausted its retries.
+              // Operators wire this to their alerting / dashboards —
+              // failed-after-retries is "needs human review" not "in the
+              // normal flow." The audit chain captures the final state.
+              if (!willRetry) {
+                _emit("system.queue.dlq.write", {
+                  metadata: {
+                    queue: queueName, backend: backend.name, jobId: job.jobId,
+                    attempts: job.attempts, traceId: job.traceId,
+                  },
+                  reason:  msg,
+                  outcome: "failure",
+                });
+              }
+              return { status: "failed", movedToDlq: !willRetry };
+            });
+        });
+    }
+  );
+}
+
+/**
+ * @primitive b.queue.tick
+ * @signature b.queue.tick(opts)
+ * @since     0.18.43
+ * @status    stable
+ * @related   b.queue.consume, b.queue.enqueue, b.queue.dlqList
+ *
+ * Drain what is due, then return. Leases at most `max` available jobs, runs
+ * `handler` over them, applies the same leasing, deterministic backoff and
+ * dead-letter transitions `b.queue.consume` applies, and settles. Nothing is
+ * left running when the promise resolves: no resident loop, no timer, no
+ * consumer registered for shutdown to wait on.
+ *
+ * Pair it with `b.queue.init({ sweepIntervalMs: 0 })`. `init` otherwise starts
+ * a 30-second expired-lease sweep timer, and in a frozen isolate that timer
+ * either never fires or fires inside a later invocation, touching the backend
+ * outside any request the platform is accounting for. `tick` sweeps once per
+ * invocation itself, so the timer buys nothing there.
+ *
+ * This is the driver for a scheduled invocation — a Cloudflare `scheduled()`
+ * handler, a Lambda on EventBridge, a Kubernetes CronJob. `consume` cannot
+ * serve those: it is a resident loop and it starts a background sweep, and such
+ * a runtime freezes between invocations, so the timer never fires and the
+ * consumer is either blocking the handler or killed mid-lease. Without a
+ * per-tick entry point the durable half — attempt accounting, backoff schedule,
+ * DLQ transition — gets reimplemented by the caller, which is the part that is
+ * genuinely hard to get right.
+ *
+ * Returns `{ leased, succeeded, failed, stale, unsettled, movedToDlq,
+ * mayHaveMore, queueDepth }`.
+ *
+ * Two of those counts are not terminal states and mean different things.
+ * `stale` is benign: the backend answered that this worker no longer owns the
+ * job, so someone else is running it. `unsettled` is not: the backend REFUSED
+ * to record the outcome — complete or fail rejected after its own retries — so
+ * the job is still inflight and this tick does not know whether the handler
+ * succeeded. Those are recovered by a later sweep, and if the handler had
+ * already succeeded its side effects run a second time, so a non-zero
+ * `unsettled` is worth alerting on rather than counting as noise.
+ *
+ * `mayHaveMore` is the tick-again signal: the batch came back full, so more
+ * may be due right now. Loop on that, not on a depth count — `queueDepth` is
+ * the queue's total active depth (pending plus inflight) and includes jobs
+ * whose `availableAt` is still in the future, so a caller looping on it would
+ * spin through empty ticks waiting for a delayed job to come due. `queueDepth`
+ * is `null` when the backend could not be asked: unknown is not empty.
+ *
+ * `stale` counts jobs this tick ran but did not settle: the handler outlived
+ * the lease, the job was swept and re-leased, and the backend's attempt fence
+ * refused the completion. Those are not failures — another worker owns them —
+ * and a non-zero `stale` means `leaseMs` is too short for the handler, or the
+ * batch too large to finish inside it.
+ *
+ * Jobs run sequentially by default. Pass `concurrency` to overlap them; the
+ * lease is held for `leaseMs` either way, so a batch that will take longer than
+ * the lease should either raise `leaseMs` or call `ctx.extendLease(ms)` from
+ * the handler.
+ *
+ * @opts
+ *   queue:       string,     // required — the queue name to drain
+ *   handler:     function,   // required — (job, ctx) => Promise, same as consume
+ *   max:         number,     // default: 10 — most jobs to lease this tick
+ *   leaseMs:     number,     // default: 30s — lease duration for the batch
+ *   concurrency: number,     // default: 1 — jobs run in parallel
+ *   backend:     string,     // named backend; default the init default
+ *
+ * @example
+ *   var result = await b.queue.tick({
+ *     queue:   "webhooks",
+ *     max:     50,
+ *     handler: async function (job) { await deliver(job.payload); },
+ *   });
+ *   while (result.mayHaveMore) result = await b.queue.tick({ queue: "webhooks", max: 50, handler: deliver });
+ *   // → { leased: 50, succeeded: 48, failed: 2, stale: 0, unsettled: 0,
+ *   //     movedToDlq: 1, mayHaveMore: true, queueDepth: 62 }
+ */
+async function tick(opts) {
+  _requireInit();
+  opts = opts || {};
+  var queueName = opts.queue;
+  if (!queueName) throw _err("MISSING_QUEUE", "tick requires opts.queue", true);
+  if (typeof opts.handler !== "function") {
+    throw _err("INVALID_HANDLER", "tick requires opts.handler to be a function", true);
+  }
+  var max = opts.max === undefined ? 10 : opts.max;
+  if (!numericChecks.isPositiveInt(max)) {
+    throw _err("BAD_MAX", "tick({ max }) must be a positive integer, got " + JSON.stringify(opts.max), true);
+  }
+  var leaseMs = opts.leaseMs === undefined ? C.TIME.seconds(30) : opts.leaseMs;
+  if (!numericChecks.isPositiveInt(leaseMs)) {
+    throw _err("BAD_LEASE", "tick({ leaseMs }) must be a positive integer, got " + JSON.stringify(opts.leaseMs), true);
+  }
+  var concurrency = opts.concurrency === undefined ? 1 : opts.concurrency;
+  if (!numericChecks.isPositiveInt(concurrency)) {
+    throw _err("BAD_CONCURRENCY", "tick({ concurrency }) must be a positive integer, got " + JSON.stringify(opts.concurrency), true);
+  }
+
+  var backend = _backendFor(opts);
+  // Same boundary b.queue.consume has: the framework-side lifecycle (lease by
+  // count, complete/fail by jobId, framework backoff and DLQ) is what `local`
+  // and `redis` implement. `sqs` is a different model — lease returns a
+  // receiptHandle the caller must thread back, redelivery and the dead-letter
+  // queue are the SQS queue's own RedrivePolicy — so driving it through here
+  // would call complete/fail without the receipt, leave every handled message
+  // in the queue, and let SQS redeliver work that already succeeded. Refuse
+  // rather than do that quietly; an SQS consumer drives lease/complete/fail
+  // directly, as lib/queue-sqs.js documents.
+  if (backend.protocol === "sqs") {
+    throw _err("TICK_UNSUPPORTED",
+      "queue.tick does not drive the 'sqs' protocol: SQS completes and fails by " +
+      "receiptHandle and owns redelivery + DLQ server-side. Drive it directly " +
+      "(lease -> handle -> complete/fail) as lib/queue-sqs.js describes.", true);
+  }
+  // Recover abandoned leases before taking a batch. A job whose holder died
+  // between leasing and completing sits in `inflight` until something re-pends
+  // it, and the thing that normally does is the 30-second sweep timer started
+  // at init — a timer that never fires in the runtime tick exists for, because
+  // the isolate is frozen between invocations. Without this, an invocation
+  // killed mid-lease strands its job permanently in exactly the deployment
+  // this driver serves. Best-effort: a sweep failure must not stop the tick
+  // from doing the work it can still do.
+  if (backend.sweepExpired) {
+    try { await backend.sweepExpired(); }
+    catch (e) { log.debug("tick-sweep-failed", { op: "sweepExpired", queue: queueName, error: e.message }); }
+  }
+  var jobs = await backend.lease(queueName, leaseMs, max);
+  jobs = jobs || [];
+
+  var result = {
+    leased:      jobs.length,
+    succeeded:   0,
+    failed:      0,
+    stale:       0,
+    unsettled:   0,
+    movedToDlq:  0,
+    // The batch came back full, so there may be more due right now. This is
+    // the tick-again signal: it is exact, free, and unlike a depth count it
+    // cannot be fooled by jobs that are not yet available.
+    mayHaveMore: jobs.length === max,
+    queueDepth:  0,
+  };
+  var next = 0;
+  async function worker() {
+    while (next < jobs.length) {
+      var job = jobs[next++];
+      observability.event("queue.lease", 1, { queueName: queueName });
+      _emit("system.queue.consume.start", {
+        metadata: { queue: queueName, backend: backend.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
+      });
+      var settled;
+      try { settled = await _runJob(backend, queueName, job, opts.handler); }
+      catch (e) {
+        // A throw here is the BACKEND refusing to record the outcome —
+        // complete or fail rejected after its own retries. The job's state was
+        // never changed, so it is still inflight and this tick does not know
+        // whether the handler succeeded. Calling that a handler failure would
+        // be wrong twice: it reports a terminal state the job never reached,
+        // and it hides an infrastructure problem inside an ordinary tally.
+        // Report it as unsettled; a later sweep re-pends the job.
+        log.debug("tick-settle-failed", { op: "_runJob", queue: queueName, jobId: job.jobId, error: e.message });
+        settled = { status: "unsettled", movedToDlq: false };
+      }
+      if (settled && settled.status === "stale") result.stale += 1;
+      else if (settled && settled.status === "unsettled") result.unsettled += 1;
+      else if (settled && settled.status === "succeeded") result.succeeded += 1;
+      else result.failed += 1;
+      if (settled && settled.movedToDlq) result.movedToDlq += 1;
+    }
+  }
+  var workers = [];
+  for (var w = 0; w < Math.min(concurrency, jobs.length); w++) workers.push(worker());
+  await Promise.all(workers);
+
+  // Depth AFTER the batch. This is the queue's total active depth — pending
+  // plus inflight — and it deliberately does NOT drive the tick-again
+  // decision: it counts jobs whose availableAt is still in the future, so a
+  // caller looping on it would spin through empty ticks waiting for a delayed
+  // job to come due. null when the backend could not be asked; unknown is not
+  // the same as empty.
+  try { result.queueDepth = await size(queueName, opts); }
+  catch (e) {
+    log.debug("tick-size-failed", { op: "size", queue: queueName, error: e.message });
+    result.queueDepth = null;
+  }
+  return result;
+}
+
 /**
  * @primitive b.queue.size
  * @signature b.queue.size(queueName, opts)
@@ -1064,6 +1307,7 @@ module.exports = {
   enqueue:            enqueue,
   enqueueFlow:        enqueueFlow,
   consume:            consume,
+  tick:               tick,
   size:               size,
   purge:              purge,
   shutdown:           shutdown,
@@ -1074,4 +1318,8 @@ module.exports = {
   PROTOCOLS:          dispatcher.protocols,
   DEFERRED_PROTOCOLS: dispatcher.deferred,
   _resetForTest:      _resetForTest,
+  // Internal — the wrapped backend object, so a test can break one of its
+  // lifecycle calls (an exhausted breaker, an unreachable store) and assert
+  // what the drivers do about it. There is no other way to reach that path.
+  _backendForTest:    _backendFor,
 };