@open-mercato/queue 0.6.8-develop.7100.1.fbf66fca35 → 0.7.0

package/src/strategies/async.ts

@@ -36,11 +36,8 @@ interface BullQueueInterface<T> {
   close: () => Promise<void>
   getJobCounts: (...states: string[]) => Promise<Record<string, number>>
   getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{
-    id?: string
     data?: T
-    failedReason?: string
     remove: () => Promise<void>
-    updateData?: (data: T) => Promise<void>
   }>>
 }
 
@@ -69,66 +66,6 @@ type BullMQOtelModule = { BullMQOtel: new (tracerName: string) => object }
 
 const REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']
 
-/**
- * The failures BullMQ records when it gives up on a job *before* handing it to the processor.
- *
- * Both are written as a `defa` (deferred failure) marker on the job, after which the next worker
- * short-circuits in `Worker.processJob` via `getUnrecoverableErrorMessage` and fails the job without
- * calling the handler. The first comes from the stalled-job script once a job's cumulative stall
- * count passes `maxStalledCount`; the second from `maxStartedAttempts`.
- *
- * Matching the reason is what tells "the queue abandoned this" from "the handler ran and threw", and
- * it is deliberately stateless: the alternative — tracking which jobs this process has entered — can
- * only answer "did the handler run *here*", which is the wrong question the moment more than one
- * worker is running. `bullmq-abandoned-reasons.test.ts` asserts these strings still exist in the
- * installed BullMQ, so an upgrade that renames them fails loudly instead of silently disabling the
- * callback.
- */
-export const ABANDONED_JOB_REASONS = [
-  'job stalled more than allowable limit',
-  'job started more than allowable limit',
-] as const
-
-function isAbandonedJobReason(message: string): boolean {
-  return (ABANDONED_JOB_REASONS as readonly string[]).includes(message)
-}
-
-/**
- * Metadata key written onto the stored job once `onJobAbandoned` has completed for it.
- *
- * BullMQ's failed set is the durable record of abandoned jobs (`removeOnFail` keeps them), so it
- * doubles as the dead-letter queue for reports: the sweep re-delivers any abandoned job that does not
- * carry this marker. The marker — not `job.remove()` — is the acknowledgement, so the failed job
- * itself survives for diagnosis.
- */
-const ABANDON_REPORT_ACK_KEY = 'abandonReportedAt'
-// NOTE for anyone adding a retry action: the marker lives inside the job's own payload envelope, so a
-// job retried from admin tooling carries it into its next life and a second abandonment of that job
-// would never be reported. A retry path must clear `metadata.abandonReportedAt` when it re-enqueues.
-
-/**
- * How often a worker re-sweeps the failed set for unacknowledged abandoned jobs.
- *
- * Override with `QUEUE_ABANDONED_SWEEP_INTERVAL_MS` to trade recovery latency against Redis chatter.
- */
-export const ABANDONED_JOB_SWEEP_INTERVAL_MS = 5 * 60 * 1000
-
-/**
- * How long `close()` waits for in-flight reports before giving up on them.
- *
- * Bounded on purpose: the hook reaches a database, and a shutdown during an infrastructure incident
- * is exactly when that write can hang rather than fail. An unbounded wait would turn a graceful
- * shutdown into a SIGKILL and skip the telemetry flush that follows it. Abandoning the wait is safe
- * because delivery is at-least-once — an unacknowledged report is re-delivered by the next worker's
- * start-up sweep, the same path that covers a process which died mid-report.
- */
-export const ABANDONED_JOB_DRAIN_TIMEOUT_MS = 5000
-
-function resolveSweepIntervalMs(): number {
-  const configured = Number.parseInt(process.env.QUEUE_ABANDONED_SWEEP_INTERVAL_MS ?? '', 10)
-  return Number.isFinite(configured) && configured > 0 ? configured : ABANDONED_JOB_SWEEP_INTERVAL_MS
-}
-
 function payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {
   if (!payload || typeof payload !== 'object') return false
   const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }
@@ -192,105 +129,11 @@ export function createAsyncQueue<T = unknown>(
   const attempts = options?.attempts ?? 3
   const lockDuration = options?.lockDuration
   const maxStalledCount = options?.maxStalledCount
-  const onJobAbandoned = options?.onJobAbandoned
   const logger = packageLogger.child({ queue: name })
 
   let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null
   let bullWorker: BullWorkerInterface | null = null
   let bullmqModule: BullMQModule | null = null
-  let abandonedSweepTimer: ReturnType<typeof setInterval> | null = null
-  let closing = false
-
-  // In-flight `onJobAbandoned` calls. Detached from the caller that started them (the 'failed'
-  // listener or the sweep), so `close()` drains them rather than letting a deploy truncate a repair
-  // mid-write. The id set stops the two callers from double-reporting a job inside one process.
-  const pendingAbandonedReports = new Set<Promise<void>>()
-  const inFlightAbandonedJobIds = new Set<string>()
-
-  type AbandonedJobRecord = {
-    id?: string
-    data?: QueuedJob<T>
-    updateData?: (data: QueuedJob<T>) => Promise<void>
-  }
-
-  // The acknowledgement that makes delivery at-least-once: written only after the callback returns,
-  // so a callback that threw or a process that died mid-report leaves the job unmarked and a later
-  // sweep retries it. Requires the driver to expose `updateData`; without it the report simply stays
-  // unacknowledged and repeats, which the idempotency contract permits.
-  async function acknowledgeAbandonedReport(job: AbandonedJobRecord): Promise<void> {
-    if (!job.data || typeof job.updateData !== 'function') return
-    await job.updateData({
-      ...job.data,
-      metadata: { ...(job.data.metadata ?? {}), [ABANDON_REPORT_ACK_KEY]: new Date().toISOString() },
-    })
-  }
-
-  function reportAbandonedJob(job: AbandonedJobRecord, reason: string): Promise<void> | null {
-    if (!onJobAbandoned) return null
-    const payload = job.data
-    if (!payload) return null
-    if (payload.metadata && payload.metadata[ABANDON_REPORT_ACK_KEY]) return null
-    if (inFlightAbandonedJobIds.has(payload.id)) return null
-    inFlightAbandonedJobIds.add(payload.id)
-
-    const jobId = job.id ?? null
-    logger.warn('Job abandoned by the queue without running its handler', { jobId, reason })
-    const report = (async () => {
-      try {
-        await onJobAbandoned(payload, { jobId, reason })
-      } catch (hookError) {
-        logger.error('onJobAbandoned handler threw; the report stays unacknowledged and the sweep will retry it', {
-          jobId,
-          err: hookError as Error,
-        })
-        return
-      }
-      try {
-        await acknowledgeAbandonedReport(job)
-      } catch (ackError) {
-        logger.error('Failed to acknowledge an abandoned-job report; the sweep may repeat it', {
-          jobId,
-          err: ackError as Error,
-        })
-      }
-    })().finally(() => {
-      inFlightAbandonedJobIds.delete(payload.id)
-    })
-    pendingAbandonedReports.add(report)
-    void report.then(() => pendingAbandonedReports.delete(report))
-    return report
-  }
-
-  // The failed set is this strategy's dead-letter queue for abandoned jobs. Enumerating it on worker
-  // start and on an interval, and re-delivering anything unacknowledged, is what upgrades the
-  // 'failed'-listener fast path from at-most-once to at-least-once: a report lost to a crash is
-  // simply still unmarked when the next sweep looks. Residual loss: `removeOnFail` caps the set, so
-  // a job evicted before any sweep sees it is gone for good.
-  async function sweepAbandonedJobs(): Promise<void> {
-    if (!onJobAbandoned || closing) return
-    try {
-      const queue = await getQueue()
-      const failedJobs = await queue.getJobs(['failed'], 0, -1)
-      // Re-checked after the awaits: a sweep already past its guard when `close()` ran would
-      // otherwise start a report the drain has stopped waiting for. The next start-up sweep
-      // re-delivers it, so stopping here loses nothing.
-      if (closing) return
-      for (const failedJob of failedJobs) {
-        // Re-checked every iteration for the same reason: a long fan-out must not outlive the drain.
-        if (closing) return
-        const reason = failedJob.failedReason ?? ''
-        if (!isAbandonedJobReason(reason)) continue
-        // Awaited one at a time. Each report opens a request container and writes to the database, and
-        // the worst case for this loop is the first worker start after the feature ships, on a
-        // deployment that has been accumulating abandoned jobs — the largest backlog, on the process
-        // least able to absorb it. The sweep is a recovery path with no latency requirement (five
-        // minutes late is its normal mode), so pacing costs nothing worth having.
-        await reportAbandonedJob(failedJob, reason)
-      }
-    } catch (sweepError) {
-      logger.error('Abandoned-job sweep failed', { err: sweepError as Error })
-    }
-  }
   // Resolved once: a BullMQOtel instance (delegate async tracing to BullMQ) or
   // undefined (use our own metadata._trace carrier instead). Memoized as the
   // in-flight promise so concurrent first-time callers share one resolution.
@@ -412,24 +255,9 @@ export function createAsyncQueue<T = unknown>(
     })
 
     bullWorker.on('failed', (job, err) => {
-      const failedJob = job as AbandonedJobRecord | undefined
+      const jobWithId = job as { id?: string } | undefined
       const error = err as Error
-      logger.error('Job failed', { jobId: failedJob?.id, err: error })
-
-      if (!onJobAbandoned) return
-      // Any other reason means a handler ran and threw. That failure is the handler's own and it has
-      // already had its chance to record it.
-      if (!isAbandonedJobReason(error?.message ?? '')) return
-      // No payload means the queue could not give us the job at all. There is nothing to hand the
-      // callback and nothing it could repair, so reporting could only ever be a false alarm — the
-      // 'Job failed' line above still records it.
-      if (!failedJob?.data) return
-
-      // The fast path: report the moment the abandonment is observed. `reportAbandonedJob` runs the
-      // callback detached with its own try/catch — this is an EventEmitter, where an unhandled
-      // rejection is fatal to the process — and acknowledges the job only afterwards, so a report
-      // lost here is retried by the sweep.
-      reportAbandonedJob(failedJob, error.message)
+      logger.error('Job failed', { jobId: jobWithId?.id, err: error })
     })
 
     // A stalled job is redelivered under the same id while the previous worker
@@ -447,17 +275,6 @@ export function createAsyncQueue<T = unknown>(
       logger.error('Worker error', { err: error })
     })
 
-    if (onJobAbandoned) {
-      // Sweep immediately so reports lost to a previous process's crash are re-delivered as soon as
-      // a worker is back, then keep re-sweeping for anything the fast path loses while running.
-      // The timer is unref'd so it never holds the process open.
-      void sweepAbandonedJobs()
-      abandonedSweepTimer = setInterval(() => {
-        void sweepAbandonedJobs()
-      }, resolveSweepIntervalMs())
-      ;(abandonedSweepTimer as unknown as { unref?: () => void }).unref?.()
-    }
-
     logger.info('Worker started', { concurrency })
 
     // For async strategy, return a sentinel result indicating worker mode
@@ -494,31 +311,10 @@ export function createAsyncQueue<T = unknown>(
   }
 
   async function close(): Promise<void> {
-    closing = true
-    if (abandonedSweepTimer) {
-      clearInterval(abandonedSweepTimer)
-      abandonedSweepTimer = null
-    }
     if (bullWorker) {
       await bullWorker.close()
       bullWorker = null
     }
-    // Drain any abandonment report still in flight, so a deploy-time shutdown cannot cut off the very
-    // repair the callback exists to perform. Bounded: the hook writes to a database, and a shutdown
-    // during an incident is exactly when that write can hang instead of failing. Giving up costs
-    // nothing permanent — an unacknowledged report is re-delivered by the next start-up sweep.
-    if (pendingAbandonedReports.size) {
-      const drained = Promise.all([...pendingAbandonedReports]).then(() => true)
-      const expired = new Promise<boolean>((resolve) => {
-        const timer = setTimeout(() => resolve(false), ABANDONED_JOB_DRAIN_TIMEOUT_MS)
-        ;(timer as unknown as { unref?: () => void }).unref?.()
-      })
-      if (!(await Promise.race([drained, expired]))) {
-        logger.warn('Abandoned-job reports still in flight at shutdown; the sweep will retry them', {
-          pending: pendingAbandonedReports.size,
-        })
-      }
-    }
     if (bullQueue) {
       await bullQueue.close()
       bullQueue = null

package/src/types.ts

@@ -99,43 +99,6 @@ export type AsyncQueueOptions = {
   lockDuration?: number
   /** Number of stalled-job recoveries BullMQ permits before failing a job. Defaults to 1. */
   maxStalledCount?: number
-  /**
-   * Called when the queue permanently gives up on a job WITHOUT its handler having run.
-   *
-   * Why this cannot be left to the handler: a queue may destroy a job before ever calling the
-   * processor — the usual cause is a job redelivered once too often, past `maxStalledCount`. The
-   * handler then never runs, never throws and never learns, and any state it created on enqueue (a
-   * run row, a progress record) is orphaned in whatever "in progress" state it was left in, with
-   * nothing to correct it.
-   *
-   * Contract:
-   * - Fires **only** for a job the queue abandoned before its handler ran. It is deliberately not a
-   *   general "job failed" hook: a handler that ran and threw owns its own outcome and has already
-   *   had the chance to record it. Reporting both would double-report and would hide the difference
-   *   between "the work failed" and "the work never started".
-   * - **At-least-once, where the backend allows it.** The strategy reports as soon as it observes
-   *   the abandonment, and also sweeps the backend's dead-job records — on worker start, then
-   *   periodically — for reports that were never acknowledged. A report is acknowledged only after
-   *   this callback returns, so a callback that threw or a process that died mid-report is retried
-   *   by a later sweep. The callback MUST therefore be idempotent, and MUST tolerate a payload it
-   *   does not recognise. Residual loss is still possible when the backend evicts its dead-job
-   *   records before any sweep sees them; state that absolutely must never be stranded should also
-   *   have a staleness check of its own at the domain level.
-   * - Not every strategy can implement it — the local strategy runs handlers in-process, so no queue
-   *   outlives a handler to abandon its job.
-   *
-   * Backend specifics (which failures count as abandonment, and how they are detected) belong to the
-   * strategy; see `strategies/async.ts`.
-   */
-  onJobAbandoned?: (payload: unknown, info: AbandonedJobInfo) => void | Promise<void>
-}
-
-/** What the queue can say about a job it gave up on. */
-export type AbandonedJobInfo = {
-  /** The queue driver's own job id, or null when it did not supply one. */
-  jobId: string | null
-  /** The failure the queue recorded, e.g. 'job stalled more than allowable limit'. */
-  reason: string
 }
 
 /**
@@ -299,26 +262,6 @@ export type WorkerMeta = {
   lockDuration?: number
   /** Number of stalled-job recoveries BullMQ permits before failing a job. */
   maxStalledCount?: number
-  /**
-   * Called when the queue abandons one of this worker's jobs without running the handler.
-   *
-   * Declared here rather than on the producing queue because the worker is the side that must hear
-   * it: the event being reported is a worker restart, and the process that restarts never
-   * constructs the enqueueing queue. See `AsyncQueueOptions.onJobAbandoned` for the contract.
-   */
-  onJobAbandoned?: AsyncQueueOptions['onJobAbandoned']
-  /**
-   * Opt-in flag allowing this queue to be selected as a user-facing scheduler
-   * target. Internal and system-only workers (webhook processors, indexers,
-   * bulk operations) must leave it unset — they stay undiscoverable by the
-   * scheduler job API.
-   */
-  schedulerSafe?: boolean
-  /**
-   * Creator features a principal must hold (on top of scheduler.jobs.manage)
-   * to schedule onto this worker's queue. Only read when schedulerSafe is set.
-   */
-  schedulerRequiredFeatures?: string[]
 }
 
 /**
@@ -338,6 +281,4 @@ export type WorkerDescriptor<T = unknown> = {
   lockDuration?: number
   /** Number of stalled-job recoveries BullMQ permits before failing a job. */
   maxStalledCount?: number
-  /** Called when the queue abandons one of this worker's jobs without running the handler. */
-  onJobAbandoned?: AsyncQueueOptions['onJobAbandoned']
 }

package/src/worker/runner.ts

@@ -24,8 +24,6 @@ export type WorkerRunnerOptions<T = unknown> = {
   lockDuration?: number
   /** Number of stalled-job recoveries BullMQ permits before failing a job. */
   maxStalledCount?: number
-  /** Called when the queue abandons a job without running the handler. */
-  onJobAbandoned?: AsyncQueueOptions['onJobAbandoned']
   /** Whether to set up graceful shutdown handlers */
   gracefulShutdown?: boolean
   /** If true, don't block - return immediately after starting processing (for multi-queue mode) */
@@ -153,7 +151,6 @@ export async function runWorker<T = unknown>(
     concurrency = 1,
     lockDuration,
     maxStalledCount,
-    onJobAbandoned,
     gracefulShutdown = true,
     background = false,
     strategy: strategyOption,
@@ -179,7 +176,6 @@ export async function runWorker<T = unknown>(
     concurrency,
     lockDuration,
     maxStalledCount,
-    onJobAbandoned,
   })
 
   // Set up graceful shutdown