@open-mercato/queue 0.6.8-develop.7015.1.af90a2ddc7 → 0.6.8-develop.7019.1.f4c01c4b5c

package/src/strategies/async.ts

@@ -36,8 +36,11 @@ interface BullQueueInterface<T> {
   close: () => Promise<void>
   getJobCounts: (...states: string[]) => Promise<Record<string, number>>
   getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{
+    id?: string
     data?: T
+    failedReason?: string
     remove: () => Promise<void>
+    updateData?: (data: T) => Promise<void>
   }>>
 }
 
@@ -66,6 +69,66 @@ type BullMQOtelModule = { BullMQOtel: new (tracerName: string) => object }
 
 const REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']
 
+/**
+ * The failures BullMQ records when it gives up on a job *before* handing it to the processor.
+ *
+ * Both are written as a `defa` (deferred failure) marker on the job, after which the next worker
+ * short-circuits in `Worker.processJob` via `getUnrecoverableErrorMessage` and fails the job without
+ * calling the handler. The first comes from the stalled-job script once a job's cumulative stall
+ * count passes `maxStalledCount`; the second from `maxStartedAttempts`.
+ *
+ * Matching the reason is what tells "the queue abandoned this" from "the handler ran and threw", and
+ * it is deliberately stateless: the alternative — tracking which jobs this process has entered — can
+ * only answer "did the handler run *here*", which is the wrong question the moment more than one
+ * worker is running. `bullmq-abandoned-reasons.test.ts` asserts these strings still exist in the
+ * installed BullMQ, so an upgrade that renames them fails loudly instead of silently disabling the
+ * callback.
+ */
+export const ABANDONED_JOB_REASONS = [
+  'job stalled more than allowable limit',
+  'job started more than allowable limit',
+] as const
+
+function isAbandonedJobReason(message: string): boolean {
+  return (ABANDONED_JOB_REASONS as readonly string[]).includes(message)
+}
+
+/**
+ * Metadata key written onto the stored job once `onJobAbandoned` has completed for it.
+ *
+ * BullMQ's failed set is the durable record of abandoned jobs (`removeOnFail` keeps them), so it
+ * doubles as the dead-letter queue for reports: the sweep re-delivers any abandoned job that does not
+ * carry this marker. The marker — not `job.remove()` — is the acknowledgement, so the failed job
+ * itself survives for diagnosis.
+ */
+const ABANDON_REPORT_ACK_KEY = 'abandonReportedAt'
+// NOTE for anyone adding a retry action: the marker lives inside the job's own payload envelope, so a
+// job retried from admin tooling carries it into its next life and a second abandonment of that job
+// would never be reported. A retry path must clear `metadata.abandonReportedAt` when it re-enqueues.
+
+/**
+ * How often a worker re-sweeps the failed set for unacknowledged abandoned jobs.
+ *
+ * Override with `QUEUE_ABANDONED_SWEEP_INTERVAL_MS` to trade recovery latency against Redis chatter.
+ */
+export const ABANDONED_JOB_SWEEP_INTERVAL_MS = 5 * 60 * 1000
+
+/**
+ * How long `close()` waits for in-flight reports before giving up on them.
+ *
+ * Bounded on purpose: the hook reaches a database, and a shutdown during an infrastructure incident
+ * is exactly when that write can hang rather than fail. An unbounded wait would turn a graceful
+ * shutdown into a SIGKILL and skip the telemetry flush that follows it. Abandoning the wait is safe
+ * because delivery is at-least-once — an unacknowledged report is re-delivered by the next worker's
+ * start-up sweep, the same path that covers a process which died mid-report.
+ */
+export const ABANDONED_JOB_DRAIN_TIMEOUT_MS = 5000
+
+function resolveSweepIntervalMs(): number {
+  const configured = Number.parseInt(process.env.QUEUE_ABANDONED_SWEEP_INTERVAL_MS ?? '', 10)
+  return Number.isFinite(configured) && configured > 0 ? configured : ABANDONED_JOB_SWEEP_INTERVAL_MS
+}
+
 function payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {
   if (!payload || typeof payload !== 'object') return false
   const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }
@@ -129,11 +192,105 @@ export function createAsyncQueue<T = unknown>(
   const attempts = options?.attempts ?? 3
   const lockDuration = options?.lockDuration
   const maxStalledCount = options?.maxStalledCount
+  const onJobAbandoned = options?.onJobAbandoned
   const logger = packageLogger.child({ queue: name })
 
   let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null
   let bullWorker: BullWorkerInterface | null = null
   let bullmqModule: BullMQModule | null = null
+  let abandonedSweepTimer: ReturnType<typeof setInterval> | null = null
+  let closing = false
+
+  // In-flight `onJobAbandoned` calls. Detached from the caller that started them (the 'failed'
+  // listener or the sweep), so `close()` drains them rather than letting a deploy truncate a repair
+  // mid-write. The id set stops the two callers from double-reporting a job inside one process.
+  const pendingAbandonedReports = new Set<Promise<void>>()
+  const inFlightAbandonedJobIds = new Set<string>()
+
+  type AbandonedJobRecord = {
+    id?: string
+    data?: QueuedJob<T>
+    updateData?: (data: QueuedJob<T>) => Promise<void>
+  }
+
+  // The acknowledgement that makes delivery at-least-once: written only after the callback returns,
+  // so a callback that threw or a process that died mid-report leaves the job unmarked and a later
+  // sweep retries it. Requires the driver to expose `updateData`; without it the report simply stays
+  // unacknowledged and repeats, which the idempotency contract permits.
+  async function acknowledgeAbandonedReport(job: AbandonedJobRecord): Promise<void> {
+    if (!job.data || typeof job.updateData !== 'function') return
+    await job.updateData({
+      ...job.data,
+      metadata: { ...(job.data.metadata ?? {}), [ABANDON_REPORT_ACK_KEY]: new Date().toISOString() },
+    })
+  }
+
+  function reportAbandonedJob(job: AbandonedJobRecord, reason: string): Promise<void> | null {
+    if (!onJobAbandoned) return null
+    const payload = job.data
+    if (!payload) return null
+    if (payload.metadata && payload.metadata[ABANDON_REPORT_ACK_KEY]) return null
+    if (inFlightAbandonedJobIds.has(payload.id)) return null
+    inFlightAbandonedJobIds.add(payload.id)
+
+    const jobId = job.id ?? null
+    logger.warn('Job abandoned by the queue without running its handler', { jobId, reason })
+    const report = (async () => {
+      try {
+        await onJobAbandoned(payload, { jobId, reason })
+      } catch (hookError) {
+        logger.error('onJobAbandoned handler threw; the report stays unacknowledged and the sweep will retry it', {
+          jobId,
+          err: hookError as Error,
+        })
+        return
+      }
+      try {
+        await acknowledgeAbandonedReport(job)
+      } catch (ackError) {
+        logger.error('Failed to acknowledge an abandoned-job report; the sweep may repeat it', {
+          jobId,
+          err: ackError as Error,
+        })
+      }
+    })().finally(() => {
+      inFlightAbandonedJobIds.delete(payload.id)
+    })
+    pendingAbandonedReports.add(report)
+    void report.then(() => pendingAbandonedReports.delete(report))
+    return report
+  }
+
+  // The failed set is this strategy's dead-letter queue for abandoned jobs. Enumerating it on worker
+  // start and on an interval, and re-delivering anything unacknowledged, is what upgrades the
+  // 'failed'-listener fast path from at-most-once to at-least-once: a report lost to a crash is
+  // simply still unmarked when the next sweep looks. Residual loss: `removeOnFail` caps the set, so
+  // a job evicted before any sweep sees it is gone for good.
+  async function sweepAbandonedJobs(): Promise<void> {
+    if (!onJobAbandoned || closing) return
+    try {
+      const queue = await getQueue()
+      const failedJobs = await queue.getJobs(['failed'], 0, -1)
+      // Re-checked after the awaits: a sweep already past its guard when `close()` ran would
+      // otherwise start a report the drain has stopped waiting for. The next start-up sweep
+      // re-delivers it, so stopping here loses nothing.
+      if (closing) return
+      for (const failedJob of failedJobs) {
+        // Re-checked every iteration for the same reason: a long fan-out must not outlive the drain.
+        if (closing) return
+        const reason = failedJob.failedReason ?? ''
+        if (!isAbandonedJobReason(reason)) continue
+        // Awaited one at a time. Each report opens a request container and writes to the database, and
+        // the worst case for this loop is the first worker start after the feature ships, on a
+        // deployment that has been accumulating abandoned jobs — the largest backlog, on the process
+        // least able to absorb it. The sweep is a recovery path with no latency requirement (five
+        // minutes late is its normal mode), so pacing costs nothing worth having.
+        await reportAbandonedJob(failedJob, reason)
+      }
+    } catch (sweepError) {
+      logger.error('Abandoned-job sweep failed', { err: sweepError as Error })
+    }
+  }
   // Resolved once: a BullMQOtel instance (delegate async tracing to BullMQ) or
   // undefined (use our own metadata._trace carrier instead). Memoized as the
   // in-flight promise so concurrent first-time callers share one resolution.
@@ -255,9 +412,24 @@ export function createAsyncQueue<T = unknown>(
     })
 
     bullWorker.on('failed', (job, err) => {
-      const jobWithId = job as { id?: string } | undefined
+      const failedJob = job as AbandonedJobRecord | undefined
       const error = err as Error
-      logger.error('Job failed', { jobId: jobWithId?.id, err: error })
+      logger.error('Job failed', { jobId: failedJob?.id, err: error })
+
+      if (!onJobAbandoned) return
+      // Any other reason means a handler ran and threw. That failure is the handler's own and it has
+      // already had its chance to record it.
+      if (!isAbandonedJobReason(error?.message ?? '')) return
+      // No payload means the queue could not give us the job at all. There is nothing to hand the
+      // callback and nothing it could repair, so reporting could only ever be a false alarm — the
+      // 'Job failed' line above still records it.
+      if (!failedJob?.data) return
+
+      // The fast path: report the moment the abandonment is observed. `reportAbandonedJob` runs the
+      // callback detached with its own try/catch — this is an EventEmitter, where an unhandled
+      // rejection is fatal to the process — and acknowledges the job only afterwards, so a report
+      // lost here is retried by the sweep.
+      reportAbandonedJob(failedJob, error.message)
     })
 
     // A stalled job is redelivered under the same id while the previous worker
@@ -275,6 +447,17 @@ export function createAsyncQueue<T = unknown>(
       logger.error('Worker error', { err: error })
     })
 
+    if (onJobAbandoned) {
+      // Sweep immediately so reports lost to a previous process's crash are re-delivered as soon as
+      // a worker is back, then keep re-sweeping for anything the fast path loses while running.
+      // The timer is unref'd so it never holds the process open.
+      void sweepAbandonedJobs()
+      abandonedSweepTimer = setInterval(() => {
+        void sweepAbandonedJobs()
+      }, resolveSweepIntervalMs())
+      ;(abandonedSweepTimer as unknown as { unref?: () => void }).unref?.()
+    }
+
     logger.info('Worker started', { concurrency })
 
     // For async strategy, return a sentinel result indicating worker mode
@@ -311,10 +494,31 @@ export function createAsyncQueue<T = unknown>(
   }
 
   async function close(): Promise<void> {
+    closing = true
+    if (abandonedSweepTimer) {
+      clearInterval(abandonedSweepTimer)
+      abandonedSweepTimer = null
+    }
     if (bullWorker) {
       await bullWorker.close()
       bullWorker = null
     }
+    // Drain any abandonment report still in flight, so a deploy-time shutdown cannot cut off the very
+    // repair the callback exists to perform. Bounded: the hook writes to a database, and a shutdown
+    // during an incident is exactly when that write can hang instead of failing. Giving up costs
+    // nothing permanent — an unacknowledged report is re-delivered by the next start-up sweep.
+    if (pendingAbandonedReports.size) {
+      const drained = Promise.all([...pendingAbandonedReports]).then(() => true)
+      const expired = new Promise<boolean>((resolve) => {
+        const timer = setTimeout(() => resolve(false), ABANDONED_JOB_DRAIN_TIMEOUT_MS)
+        ;(timer as unknown as { unref?: () => void }).unref?.()
+      })
+      if (!(await Promise.race([drained, expired]))) {
+        logger.warn('Abandoned-job reports still in flight at shutdown; the sweep will retry them', {
+          pending: pendingAbandonedReports.size,
+        })
+      }
+    }
     if (bullQueue) {
       await bullQueue.close()
       bullQueue = null

package/src/types.ts

@@ -99,6 +99,43 @@ export type AsyncQueueOptions = {
   lockDuration?: number
   /** Number of stalled-job recoveries BullMQ permits before failing a job. Defaults to 1. */
   maxStalledCount?: number
+  /**
+   * Called when the queue permanently gives up on a job WITHOUT its handler having run.
+   *
+   * Why this cannot be left to the handler: a queue may destroy a job before ever calling the
+   * processor — the usual cause is a job redelivered once too often, past `maxStalledCount`. The
+   * handler then never runs, never throws and never learns, and any state it created on enqueue (a
+   * run row, a progress record) is orphaned in whatever "in progress" state it was left in, with
+   * nothing to correct it.
+   *
+   * Contract:
+   * - Fires **only** for a job the queue abandoned before its handler ran. It is deliberately not a
+   *   general "job failed" hook: a handler that ran and threw owns its own outcome and has already
+   *   had the chance to record it. Reporting both would double-report and would hide the difference
+   *   between "the work failed" and "the work never started".
+   * - **At-least-once, where the backend allows it.** The strategy reports as soon as it observes
+   *   the abandonment, and also sweeps the backend's dead-job records — on worker start, then
+   *   periodically — for reports that were never acknowledged. A report is acknowledged only after
+   *   this callback returns, so a callback that threw or a process that died mid-report is retried
+   *   by a later sweep. The callback MUST therefore be idempotent, and MUST tolerate a payload it
+   *   does not recognise. Residual loss is still possible when the backend evicts its dead-job
+   *   records before any sweep sees them; state that absolutely must never be stranded should also
+   *   have a staleness check of its own at the domain level.
+   * - Not every strategy can implement it — the local strategy runs handlers in-process, so no queue
+   *   outlives a handler to abandon its job.
+   *
+   * Backend specifics (which failures count as abandonment, and how they are detected) belong to the
+   * strategy; see `strategies/async.ts`.
+   */
+  onJobAbandoned?: (payload: unknown, info: AbandonedJobInfo) => void | Promise<void>
+}
+
+/** What the queue can say about a job it gave up on. */
+export type AbandonedJobInfo = {
+  /** The queue driver's own job id, or null when it did not supply one. */
+  jobId: string | null
+  /** The failure the queue recorded, e.g. 'job stalled more than allowable limit'. */
+  reason: string
 }
 
 /**
@@ -262,6 +299,14 @@ export type WorkerMeta = {
   lockDuration?: number
   /** Number of stalled-job recoveries BullMQ permits before failing a job. */
   maxStalledCount?: number
+  /**
+   * Called when the queue abandons one of this worker's jobs without running the handler.
+   *
+   * Declared here rather than on the producing queue because the worker is the side that must hear
+   * it: the event being reported is a worker restart, and the process that restarts never
+   * constructs the enqueueing queue. See `AsyncQueueOptions.onJobAbandoned` for the contract.
+   */
+  onJobAbandoned?: AsyncQueueOptions['onJobAbandoned']
 }
 
 /**
@@ -281,4 +326,6 @@ export type WorkerDescriptor<T = unknown> = {
   lockDuration?: number
   /** Number of stalled-job recoveries BullMQ permits before failing a job. */
   maxStalledCount?: number
+  /** Called when the queue abandons one of this worker's jobs without running the handler. */
+  onJobAbandoned?: AsyncQueueOptions['onJobAbandoned']
 }

package/src/worker/runner.ts

@@ -24,6 +24,8 @@ export type WorkerRunnerOptions<T = unknown> = {
   lockDuration?: number
   /** Number of stalled-job recoveries BullMQ permits before failing a job. */
   maxStalledCount?: number
+  /** Called when the queue abandons a job without running the handler. */
+  onJobAbandoned?: AsyncQueueOptions['onJobAbandoned']
   /** Whether to set up graceful shutdown handlers */
   gracefulShutdown?: boolean
   /** If true, don't block - return immediately after starting processing (for multi-queue mode) */
@@ -151,6 +153,7 @@ export async function runWorker<T = unknown>(
     concurrency = 1,
     lockDuration,
     maxStalledCount,
+    onJobAbandoned,
     gracefulShutdown = true,
     background = false,
     strategy: strategyOption,
@@ -176,6 +179,7 @@ export async function runWorker<T = unknown>(
     concurrency,
     lockDuration,
     maxStalledCount,
+    onJobAbandoned,
   })
 
   // Set up graceful shutdown