@open-mercato/queue 0.6.7 → 0.6.8-develop.6875.1.871a4afc94

package/src/pending-probe.ts

@@ -149,14 +149,16 @@ async function probeAsyncQueue(
     return errorResult(queueName, 'async', new Error('bullmq is not installed'))
   }
 
-  const { getRedisUrl } = await import('@open-mercato/shared/lib/redis/connection')
+  const { getRedisUrl, parseRedisUrl } = await import('@open-mercato/shared/lib/redis/connection')
   let connection = options?.connection
   if (!connection) {
     const url = getRedisUrl('QUEUE')
     if (!url) {
       return errorResult(queueName, 'async', new Error('QUEUE Redis URL is not configured'))
     }
-    connection = { url }
+    connection = parseRedisUrl(url)
+  } else if (connection.url) {
+    connection = parseRedisUrl(connection.url)
   }
 
   let queue: InstanceType<BullMQModuleShape['Queue']> | null = null

package/src/strategies/async.ts

@@ -1,5 +1,7 @@
 import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'
-import { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'
+import { getRedisUrlOrThrow, parseRedisUrl } from '@open-mercato/shared/lib/redis/connection'
+import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
+import { attachTraceMetadata, runJobInTrace } from '../tracing'
 import { createLogger } from '@open-mercato/shared/lib/logger'
 
 const packageLogger = createLogger('queue')
@@ -7,13 +9,13 @@ const packageLogger = createLogger('queue')
 // BullMQ interface types - we define the shape we use to maintain type safety
 // while keeping bullmq as an optional peer dependency
 type ConnectionOptions = {
-  url?: string
   host?: string
   port?: number
   username?: string
   password?: string
   db?: number
   tls?: Record<string, unknown>
+  family?: number
 }
 
 interface BullQueueInterface<T> {
@@ -43,14 +45,23 @@ interface BullWorkerInterface {
 }
 
 interface BullMQModule {
-  Queue: new <T>(name: string, opts: { connection: ConnectionOptions }) => BullQueueInterface<T>
+  Queue: new <T>(name: string, opts: { connection: ConnectionOptions; telemetry?: unknown }) => BullQueueInterface<T>
   Worker: new <T>(
     name: string,
     processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,
-    opts: { connection: ConnectionOptions; concurrency: number }
+    opts: {
+      connection: ConnectionOptions
+      concurrency: number
+      telemetry?: unknown
+      lockDuration?: number
+      maxStalledCount?: number
+    }
   ) => BullWorkerInterface
 }
 
+/** The `bullmq-otel` package (optional). Loaded only when an OTLP backend is active. */
+type BullMQOtelModule = { BullMQOtel: new (tracerName: string) => object }
+
 const REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']
 
 function payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {
@@ -69,13 +80,13 @@ function payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {
 /**
  * Resolves Redis connection options from various sources.
  *
- * BullMQ expects an ioredis-compatible connection object. Preserve the full
- * Redis URL under the `url` key so rediss://, username, database, and query
- * params are not lost in translation.
+ * BullMQ expects ioredis connection fields rather than a nested URL string.
+ * Parse URL-based configuration at this boundary while keeping the public
+ * queue API compatible with existing `{ url }` callers.
  */
 function resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {
   if (options?.url) {
-    return { url: options.url }
+    return parseRedisUrl(options.url)
   }
 
   if (options?.host) {
@@ -86,10 +97,11 @@ function resolveConnection(options?: AsyncQueueOptions['connection']): Connectio
       password: options.password,
       db: options.db,
       tls: options.tls,
+      family: options.family,
     }
   }
 
-  return { url: getRedisUrlOrThrow('QUEUE') }
+  return parseRedisUrl(getRedisUrlOrThrow('QUEUE'))
 }
 
 /**
@@ -111,11 +123,18 @@ export function createAsyncQueue<T = unknown>(
 ): Queue<T> {
   const connection = resolveConnection(options?.connection)
   const concurrency = options?.concurrency ?? 1
+  const attempts = options?.attempts ?? 3
+  const lockDuration = options?.lockDuration
+  const maxStalledCount = options?.maxStalledCount
   const logger = packageLogger.child({ queue: name })
 
   let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null
   let bullWorker: BullWorkerInterface | null = null
   let bullmqModule: BullMQModule | null = null
+  // Resolved once: a BullMQOtel instance (delegate async tracing to BullMQ) or
+  // undefined (use our own metadata._trace carrier instead). Memoized as the
+  // in-flight promise so concurrent first-time callers share one resolution.
+  let telemetryPromise: Promise<object | undefined> | null = null
 
   // -------------------------------------------------------------------------
   // Lazy BullMQ initialization
@@ -134,10 +153,35 @@ export function createAsyncQueue<T = unknown>(
     return bullmqModule
   }
 
+  /**
+   * When an OTLP backend is active, delegate async-queue tracing to `bullmq-otel`
+   * (richer BullMQ-internal spans: add / process / wait / attempts). Returns
+   * `undefined` — meaning "use our own `metadata._trace` carrier" — when telemetry
+   * is off, a non-OTEL backend is selected, or `bullmq-otel` isn't installed. (The
+   * `local` strategy always uses our carrier; it isn't BullMQ, so `bullmq-otel`
+   * cannot instrument it.)
+   */
+  async function getQueueTelemetry(): Promise<object | undefined> {
+    if (!telemetryPromise) {
+      telemetryPromise = (async () => {
+        if (!getTelemetryRuntime()?.canUseGlobalTracePropagation()) return undefined
+        try {
+          const mod = (await import('bullmq-otel')) as unknown as BullMQOtelModule
+          return new mod.BullMQOtel('open-mercato')
+        } catch {
+          packageLogger.warn('bullmq-otel not available; using built-in trace carrier', { queue: name })
+          return undefined
+        }
+      })()
+    }
+    return telemetryPromise
+  }
+
   async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {
     if (!bullQueue) {
       const { Queue: BullQueueClass } = await getBullMQ()
-      bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection })
+      const telemetry = await getQueueTelemetry()
+      bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection, ...(telemetry ? { telemetry } : {}) })
     }
     return bullQueue
   }
@@ -148,17 +192,21 @@ export function createAsyncQueue<T = unknown>(
 
   async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {
     const queue = await getQueue()
+    // When bullmq-otel handles propagation, don't also attach our carrier.
+    const telemetry = await getQueueTelemetry()
+    const metadata = telemetry ? undefined : attachTraceMetadata(undefined)
     const jobData: QueuedJob<T> = {
       id: crypto.randomUUID(),
       payload: data,
       createdAt: new Date().toISOString(),
+      ...(metadata ? { metadata } : {}),
     }
 
     const job = await queue.add(jobData.id, jobData, {
       delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,
       removeOnComplete: true,
       removeOnFail: 1000,
-      attempts: 3,
+      attempts,
       backoff: { type: 'exponential', delay: 1000 },
     })
 
@@ -167,21 +215,33 @@ export function createAsyncQueue<T = unknown>(
 
   async function process(handler: JobHandler<T>): Promise<ProcessResult> {
     const { Worker } = await getBullMQ()
+    const telemetry = await getQueueTelemetry()
 
     // Create worker that processes jobs
     bullWorker = new Worker<QueuedJob<T>>(
       name,
       async (job) => {
         const jobData = job.data
-        await handler(jobData, {
+        const ctx = {
           jobId: job.id ?? jobData.id,
           attemptNumber: job.attemptsMade + 1,
           queueName: name,
-        })
+        }
+        // With bullmq-otel active, BullMQ owns the process span and active
+        // context (the handler's pg/undici spans nest under it). Otherwise
+        // continue the trace from our own carrier.
+        if (telemetry) {
+          await handler(jobData, ctx)
+        } else {
+          await runJobInTrace(name, jobData.metadata, () => handler(jobData, ctx))
+        }
       },
       {
         connection,
         concurrency,
+        ...(telemetry ? { telemetry } : {}),
+        ...(lockDuration !== undefined ? { lockDuration } : {}),
+        ...(maxStalledCount !== undefined ? { maxStalledCount } : {}),
       }
     )
 
@@ -197,6 +257,16 @@ export function createAsyncQueue<T = unknown>(
       logger.error('Job failed', { jobId: jobWithId?.id, err: error })
     })
 
+    // A stalled job is redelivered under the same id while the previous worker
+    // may still be running it, so this is the signal that a handler is about to
+    // be executed twice. BullMQ's docs require surfacing it: without this line
+    // duplicate processing is invisible.
+    bullWorker.on('stalled', (jobId) => {
+      logger.warn('Job stalled and will be redelivered — the handler may run concurrently with a previous delivery', {
+        jobId: typeof jobId === 'string' ? jobId : null,
+      })
+    })
+
     bullWorker.on('error', (err) => {
       const error = err as Error
       logger.error('Worker error', { err: error })

package/src/strategies/local.ts

@@ -3,6 +3,7 @@ import path from 'node:path'
 import crypto from 'node:crypto'
 import { createLogger } from '@open-mercato/shared/lib/logger'
 import type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'
+import { attachTraceMetadata, runJobInTrace } from '../tracing'
 
 const packageLogger = createLogger('queue')
 
@@ -49,9 +50,18 @@ const fsp = fs.promises
  * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)
  * - Not suitable for production or multi-process environments
  *
- * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential
- * backoff and moved to a dead-letter store once attempts are exhausted (see the
- * retry handling in `process()` below).
+ * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.
+ * **This strategy keeps no failed-job store**: once attempts are exhausted the job is
+ * removed from `queue.json` and only counted in `state.failedCount`, so the payload is
+ * lost and the failure survives solely as an error log line. The `async` strategy keeps
+ * only a bounded inspection window: `removeOnFail: 1000` retains the most recent 1000
+ * failures and removes older ones as later failures arrive. Workflows that require
+ * no-loss persistence must write their own durable record before enqueueing, regardless
+ * of strategy.
+ *
+ * `DEFAULT_MAX_ATTEMPTS` is a module constant, not a per-job option — callers cannot
+ * request a different attempt count. (`async` likewise hard-codes `attempts: 3`.)
+ * See the retry handling in `process()` below.
  *
  * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not
  * block the Node.js event loop. A per-queue promise chain serializes
@@ -196,11 +206,13 @@ export function createLocalQueue<T = unknown>(
     const availableAt = options?.delayMs && options.delayMs > 0
       ? new Date(Date.now() + options.delayMs).toISOString()
       : undefined
+    const metadata = attachTraceMetadata(undefined)
     const job: StoredJob<T> = {
       id: generateId(),
       payload: data,
       createdAt: new Date().toISOString(),
       ...(availableAt ? { availableAt } : {}),
+      ...(metadata ? { metadata } : {}),
     }
     await withFileLock(async () => {
       const jobs = await readQueue()
@@ -246,12 +258,14 @@ export function createLocalQueue<T = unknown>(
       for (const job of jobsToProcess) {
         const attemptNumber = (job.attemptCount ?? 0) + 1
         try {
-          await Promise.resolve(
-            handler(job, {
-              jobId: job.id,
-              attemptNumber,
-              queueName: name,
-            })
+          await runJobInTrace(name, job.metadata, () =>
+            Promise.resolve(
+              handler(job, {
+                jobId: job.id,
+                attemptNumber,
+                queueName: name,
+              })
+            )
           )
           processed++
           lastJobId = job.id
@@ -262,7 +276,7 @@ export function createLocalQueue<T = unknown>(
           failed++
           lastJobId = job.id
           if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {
-            logger.error('Job exhausted all attempts, moving to dead letter', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })
+            logger.error('Job exhausted all attempts; dropping it (no dead-letter store)', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })
             deadJobIds.add(job.id)
           } else {
             const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)

package/src/tracing.ts

@@ -0,0 +1,58 @@
+import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
+
+/**
+ * Distributed-trace propagation across the enqueue → worker boundary.
+ *
+ * The W3C trace carrier rides on the job's `metadata._trace` (a first-class
+ * metadata channel, not the user payload). Both halves are automatic — the
+ * strategies call `attachTraceMetadata` at enqueue and `runJobInTrace` at
+ * dispatch — so a worker joins the enqueuing request's trace with no per-worker
+ * code. Everything here is a cheap no-op when telemetry is off.
+ *
+ * This also covers anything that rides the queue: persistent event subscribers
+ * (the event bus enqueues) and outbound webhook delivery (queued) become part of
+ * the originating request's trace for free.
+ */
+const TRACE_META_KEY = '_trace'
+
+/**
+ * Attach the active trace context to a job's metadata. Returns `metadata`
+ * unchanged when telemetry is off (no active span → empty carrier), so jobs stay
+ * clean unless tracing is active.
+ */
+export function attachTraceMetadata(
+  metadata: Record<string, unknown> | undefined,
+): Record<string, unknown> | undefined {
+  const carrier = getTelemetryRuntime()?.captureTraceContext() ?? {}
+  if (Object.keys(carrier).length === 0) return metadata
+  return { ...(metadata ?? {}), [TRACE_META_KEY]: carrier }
+}
+
+/**
+ * Run a job handler inside a span (`queue.<queueName>`) that continues the
+ * producer's trace from the carrier on `metadata`. With no carrier (or telemetry
+ * off) it runs `fn` under a fresh root span — and a no-op when off. The span
+ * ends when `fn` settles (sync or async).
+ */
+export function runJobInTrace<T>(
+  queueName: string,
+  metadata: Record<string, unknown> | undefined,
+  fn: () => T,
+): T {
+  const runtime = getTelemetryRuntime()
+  if (!runtime) return fn()
+  return runtime.continueTrace(
+    readTraceCarrier(metadata),
+    `queue.${queueName}`,
+    fn,
+    { kind: 'consumer' },
+  )
+}
+
+function readTraceCarrier(
+  metadata: Record<string, unknown> | undefined,
+): Record<string, string> | undefined {
+  const raw = metadata?.[TRACE_META_KEY]
+  if (!raw || typeof raw !== 'object') return undefined
+  return raw as Record<string, string>
+}

package/src/types.ts

@@ -81,6 +81,8 @@ export type RedisConnectionOptions = {
   db?: number
   /** TLS configuration for rediss / encrypted Redis */
   tls?: Record<string, unknown>
+  /** IP family used by Redis DNS resolution */
+  family?: number
 }
 
 /**
@@ -91,6 +93,12 @@ export type AsyncQueueOptions = {
   connection?: RedisConnectionOptions
   /** Number of concurrent job processors. Defaults to 1 */
   concurrency?: number
+  /** Number of attempts for newly enqueued jobs. Defaults to 3. */
+  attempts?: number
+  /** How long a job lock is held before the job counts as stalled, in ms. Defaults to 30000. */
+  lockDuration?: number
+  /** Number of stalled-job recoveries BullMQ permits before failing a job. Defaults to 1. */
+  maxStalledCount?: number
 }
 
 /**
@@ -250,6 +258,10 @@ export type WorkerMeta = {
   id?: string
   /** Worker concurrency (default: 1) */
   concurrency?: number
+  /** How long a job lock is held before the job counts as stalled, in ms. */
+  lockDuration?: number
+  /** Number of stalled-job recoveries BullMQ permits before failing a job. */
+  maxStalledCount?: number
 }
 
 /**
@@ -265,4 +277,8 @@ export type WorkerDescriptor<T = unknown> = {
   handler: JobHandler<T>
   /** Concurrency level */
   concurrency: number
+  /** How long a job lock is held before the job counts as stalled, in ms. */
+  lockDuration?: number
+  /** Number of stalled-job recoveries BullMQ permits before failing a job. */
+  maxStalledCount?: number
 }

package/src/worker/runner.ts

@@ -1,5 +1,9 @@
 import { createQueue } from '../factory'
 import { createLogger } from '@open-mercato/shared/lib/logger'
+import {
+  getTelemetryRuntime,
+  isTelemetryBackendEnabled,
+} from '@open-mercato/shared/lib/telemetry/runtime'
 import type { Queue, JobHandler, AsyncQueueOptions, QueueStrategyType } from '../types'
 
 const logger = createLogger('queue').child({ component: 'worker' })
@@ -16,6 +20,10 @@ export type WorkerRunnerOptions<T = unknown> = {
   connection?: AsyncQueueOptions['connection']
   /** Number of concurrent jobs to process */
   concurrency?: number
+  /** How long a job lock is held before the job counts as stalled, in ms. */
+  lockDuration?: number
+  /** Number of stalled-job recoveries BullMQ permits before failing a job. */
+  maxStalledCount?: number
   /** Whether to set up graceful shutdown handlers */
   gracefulShutdown?: boolean
   /** If true, don't block - return immediately after starting processing (for multi-queue mode) */
@@ -67,6 +75,17 @@ function registerShutdownHandlers(): void {
     unregisterShutdownHandlers(sigtermHandler, sigintHandler)
     shutdownInProgress = false
 
+    // Flush buffered spans/logs before the process dies. A worker never returns
+    // from run(), so bin.ts's post-run shutdownTelemetry() is unreachable on this
+    // path — without this, the BatchSpanProcessor's ~5s tail is dropped on every
+    // restart/redeploy. Idempotent and a no-op when telemetry is off; a flush
+    // failure must not turn a clean shutdown into a failed one.
+    try {
+      await getTelemetryRuntime()?.shutdown()
+    } catch (error) {
+      logger.error('Error flushing telemetry during shutdown', { err: error })
+    }
+
     if (!hasError) {
       logger.info('Worker closed successfully')
     }
@@ -130,11 +149,22 @@ export async function runWorker<T = unknown>(
     handler,
     connection,
     concurrency = 1,
+    lockDuration,
+    maxStalledCount,
     gracefulShutdown = true,
     background = false,
     strategy: strategyOption,
   } = options
 
+  // Worker processes don't run Next's instrumentation hook, so initialize
+  // telemetry here — this is the single bootstrap every standalone worker passes
+  // through. Import the telemetry package only for an explicit enabled backend;
+  // with the default/unset backend the worker never evaluates the package.
+  if (!getTelemetryRuntime() && isTelemetryBackendEnabled()) {
+    const { initTelemetry } = await import('@open-mercato/telemetry')
+    await initTelemetry()
+  }
+
   // Determine queue strategy from option, env var, or default to 'local'
   const strategy: QueueStrategyType = strategyOption
     ?? (process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local')
@@ -144,6 +174,8 @@ export async function runWorker<T = unknown>(
   const queue = createQueue<T>(queueName, strategy, {
     connection,
     concurrency,
+    lockDuration,
+    maxStalledCount,
   })
 
   // Set up graceful shutdown