@furystack/redis-task-runner 1.0.0

package/src/redis-queue-adapter.ts

@@ -0,0 +1,733 @@
+import type {
+  ClaimedTask,
+  ClaimOutcome,
+  EnqueueInput,
+  IdempotencyLeaseInput,
+  QueueAdapter,
+  QueueAdapterCapabilities,
+  WorkerSubscription,
+} from '@furystack/task-runner'
+import type { RedisClientType } from 'redis'
+
+type RedisLikeClient = RedisClientType
+
+/**
+ * Options accepted by {@link RedisQueueAdapter}.
+ */
+export type RedisQueueAdapterOptions = {
+  /**
+   * Caller-owned `redis` client. Must be connected before the adapter
+   * is used. The adapter never closes it — lifecycle stays with the
+   * application, mirroring `@furystack/redis-cross-node-bus` and
+   * `@furystack/s3-blob-store`.
+   */
+  client: RedisLikeClient
+  /**
+   * Logical service name. Used in telemetry attribution and (when
+   * `topicPrefix` is left default) helps debug shared brokers.
+   */
+  serviceName: string
+  /**
+   * Wire-level prefix applied to every stream key. Multi-service
+   * deployments pick a distinct prefix per service.
+   */
+  topicPrefix?: string
+  /**
+   * Stable consumer group name. All workers join this group; PRD §7.4
+   * relies on consumer-group competing-consumer semantics for fan-out.
+   * Default: `runner`.
+   */
+  consumerGroup?: string
+  /**
+   * Default visibility timeout (milliseconds). A handler attempt is
+   * considered stale when its PEL idle time exceeds this; XAUTOCLAIM
+   * reclaims it for another consumer. Apps can override per task type
+   * via {@link RedisQueueAdapterOptions.visibilityTimeoutByType}.
+   * Default: `60_000`.
+   */
+  visibilityTimeoutMs?: number
+  /**
+   * Per-task-type visibility-timeout override map. Useful when one
+   * type does GPU-bound work and another short polls.
+   */
+  visibilityTimeoutByType?: Record<string, number>
+  /**
+   * `XREADGROUP BLOCK` timeout (milliseconds). Lower values reduce
+   * wakeup latency on shutdown; higher values reduce broker chatter.
+   * Default: `200`.
+   */
+  blockTimeoutMs?: number
+  /**
+   * Backoff between `XREADGROUP` attempts after a transport error so
+   * a disconnected broker does not produce a tight reconnect spin.
+   * Default: `250`.
+   */
+  retryBackoffMs?: number
+  /**
+   * Idempotency-lease TTL (seconds) for
+   * {@link RedisQueueAdapter.acquireIdempotencyLease}. Default: 24h.
+   */
+  idempotencyTtlSec?: number
+  /**
+   * Delayed-dispatch tick interval (milliseconds). Each tick atomically
+   * pops due entries from the scheduler ZSET and `XADD`s them onto the
+   * matching `(type, version)` stream. Lower values = lower delay
+   * jitter at the cost of broker chatter; higher values = looser
+   * floor on `notBefore` granularity. Default: `250`.
+   */
+  schedulerIntervalMs?: number
+  /**
+   * Fleet-wide max-concurrent claims per task type. A claim whose lane
+   * is at its cap is released back to the stream and retried, so the
+   * limit holds across every worker process sharing the broker (PRD
+   * §11 fleet cap). Types absent from the map are uncapped. Enforced by
+   * a per-type ZSET of holder tokens scored by expiry (crash-safe TTL
+   * tied to the type's visibility timeout).
+   */
+  concurrencyLimits?: Record<string, number>
+}
+
+const CAPABILITIES: QueueAdapterCapabilities = Object.freeze({
+  persistent: true,
+  distributed: true,
+  delayedDispatch: true,
+  fleetCapEnforcement: true,
+  brokerSideReclaim: true,
+})
+
+const DEFAULT_GROUP = 'runner'
+const DEFAULT_VISIBILITY_MS = 60_000
+const DEFAULT_BLOCK_MS = 200
+const DEFAULT_RETRY_BACKOFF_MS = 250
+const DEFAULT_IDEMPOTENCY_TTL_SEC = 86_400
+const DEFAULT_SCHEDULER_INTERVAL_MS = 250
+const SCHEDULER_BATCH_LIMIT = 64
+
+/**
+ * Atomically: pop ZSET members whose score ≤ now, parse their JSON
+ * envelope, XADD the resulting message to the per-`(type, version)`
+ * stream, and ZREM the member. Single round-trip prevents two adapter
+ * instances from double-dispatching the same delayed task.
+ *
+ * `KEYS[1]` — scheduler ZSET key.
+ * `ARGV[1]` — current epoch ms (caller-supplied so the script's own
+ * server-clock drift cannot diverge from the runner's notion of `now`).
+ * `ARGV[2]` — batch limit.
+ * `ARGV[3]` — `topicPrefix` used to build the destination stream key.
+ *
+ * Returns the number of dispatched entries.
+ */
+const SCHEDULER_DISPATCH_SCRIPT = `
+local now = tonumber(ARGV[1])
+local limit = tonumber(ARGV[2])
+local prefix = ARGV[3]
+local sched = KEYS[1]
+
+local members = redis.call('ZRANGEBYSCORE', sched, '-inf', now, 'LIMIT', 0, limit)
+local dispatched = 0
+for _, member in ipairs(members) do
+  local removed = redis.call('ZREM', sched, member)
+  if removed == 1 then
+    local meta = cjson.decode(member)
+    local stream = prefix .. 'tasks:queue:' .. meta.type .. ':v' .. tostring(meta.handlerVersion)
+    redis.call('XADD', stream, '*',
+      'taskId', meta.taskId,
+      'type', meta.type,
+      'handlerVersion', tostring(meta.handlerVersion))
+    dispatched = dispatched + 1
+  end
+end
+return dispatched
+`
+
+/**
+ * Atomically admit one holder to a fleet-cap lane. Evicts expired
+ * holders (score ≤ now), then admits the caller's token only if the
+ * lane is below its cap.
+ *
+ * `KEYS[1]` — per-type cap ZSET key.
+ * `ARGV[1]` — current epoch ms (used to evict expired holders).
+ * `ARGV[2]` — cap (max concurrent holders).
+ * `ARGV[3]` — caller's holder token.
+ * `ARGV[4]` — holder expiry epoch ms (now + visibility timeout). The
+ * TTL frees a crashed holder's slot exactly when its claim becomes
+ * reclaimable.
+ *
+ * Returns `1` when admitted, `0` when the lane is full.
+ */
+const CAP_ACQUIRE_SCRIPT = `
+local now = tonumber(ARGV[1])
+local cap = tonumber(ARGV[2])
+redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
+if redis.call('ZCARD', KEYS[1]) < cap then
+  redis.call('ZADD', KEYS[1], ARGV[4], ARGV[3])
+  return 1
+end
+return 0
+`
+
+type RedisReceipt = {
+  stream: string
+  msgId: string
+  consumer: string
+  deliveryCount: number
+}
+
+type StreamMessageEntry = { id: string; message: Record<string, string> }
+
+type XReadGroupReply = Array<{
+  name: string
+  messages: StreamMessageEntry[]
+}>
+
+type XAutoClaimReply = {
+  nextId: string
+  messages: Array<StreamMessageEntry | null>
+}
+
+type SubscriptionState = {
+  subscription: WorkerSubscription
+  consumers: string[]
+  streams: string[]
+  abortController: AbortController
+  slotPromises: Array<Promise<void>>
+}
+
+/**
+ * Redis Streams implementation of {@link QueueAdapter}. Each
+ * `(type, handlerVersion)` lane gets its own stream
+ * (`${topicPrefix}tasks:queue:${type}:v${handlerVersion}`); workers
+ * register against the streams whose `(type, version)` they support
+ * via the {@link WorkerSubscription.compatibleVersions} map.
+ *
+ * **Concurrency.** Each subscription spawns
+ * {@link WorkerSubscription.concurrency} parallel claim slots. A slot
+ * is one consumer in the consumer group named
+ * `${workerId}-slot-${index}`; messages handed to a slot live in that
+ * consumer's PEL until ack. Multiple slots share the same group, so
+ * Redis itself enforces competing-consumer semantics.
+ *
+ * **Visibility reclaim.** Every slot iteration runs `XAUTOCLAIM` first
+ * (idle threshold = the type's `visibilityTimeoutMs`); reclaimed
+ * entries are delivered through the same `onClaim` path as fresh
+ * messages, with an attempt re-classification done by the runner core
+ * (the dataset still records the prior attempt as `'timed-out'`).
+ *
+ * **Heartbeat.** {@link RedisQueueAdapter.heartbeat} resets the PEL
+ * idle counter via `XCLAIM JUSTID` so long-running handlers don't get
+ * reclaimed, and refreshes the holder's fleet-cap lease.
+ *
+ * **Delayed dispatch.** `notBefore` parks the entry in a per-prefix
+ * scheduler ZSET; a timer atomically pops due entries onto the matching
+ * stream (`delayedDispatch: true`).
+ *
+ * **Fleet cap.** When a type appears in
+ * {@link RedisQueueAdapterOptions.concurrencyLimits}, each claim must
+ * admit a holder token into a per-type ZSET (scored by expiry) before
+ * the handler runs; a claim that finds its lane full is released back to
+ * the stream and retried, so the cap holds fleet-wide
+ * (`fleetCapEnforcement: true`).
+ */
+export class RedisQueueAdapter implements QueueAdapter {
+  public readonly capabilities: QueueAdapterCapabilities = CAPABILITIES
+
+  readonly #client: RedisLikeClient
+  readonly #topicPrefix: string
+  readonly #group: string
+  readonly #defaultVisibilityMs: number
+  readonly #visibilityByType: Record<string, number>
+  readonly #blockMs: number
+  readonly #retryBackoffMs: number
+  readonly #idempotencyTtlSec: number
+  readonly #schedulerIntervalMs: number
+  readonly #concurrencyLimits: Record<string, number>
+
+  readonly #subscriptions = new Set<SubscriptionState>()
+  readonly #ensuredGroups = new Set<string>()
+
+  /**
+   * Dedicated connection for the blocking `XREADGROUP` / `XAUTOCLAIM`
+   * slot loops. A blocking command holds the connection's reply stream
+   * until it returns, so sharing the main client would stall every store
+   * read/write, `XADD`, and `XACK` behind an in-flight `BLOCK` for up to
+   * `blockTimeoutMs` (head-of-line blocking). Duplicating mirrors
+   * `RedisCrossNodeBus`; the caller's client stays free for everything else.
+   */
+  readonly #readClient: RedisLikeClient
+  readonly #readConnectPromise: Promise<void>
+
+  readonly #schedulerTimer: ReturnType<typeof setInterval>
+  #schedulerRunning = false
+  #disposed = false
+
+  constructor(options: RedisQueueAdapterOptions) {
+    if (!options.serviceName || options.serviceName.length === 0) {
+      throw new Error('RedisQueueAdapter: `serviceName` is required')
+    }
+    this.#client = options.client
+    this.#topicPrefix = options.topicPrefix ?? ''
+    this.#group = options.consumerGroup ?? DEFAULT_GROUP
+    this.#defaultVisibilityMs = options.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_MS
+    this.#visibilityByType = options.visibilityTimeoutByType ?? {}
+    this.#blockMs = options.blockTimeoutMs ?? DEFAULT_BLOCK_MS
+    this.#retryBackoffMs = options.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS
+    this.#idempotencyTtlSec = options.idempotencyTtlSec ?? DEFAULT_IDEMPOTENCY_TTL_SEC
+    this.#schedulerIntervalMs = options.schedulerIntervalMs ?? DEFAULT_SCHEDULER_INTERVAL_MS
+    this.#concurrencyLimits = options.concurrencyLimits ?? {}
+
+    this.#readClient = options.client.duplicate()
+    this.#readConnectPromise = this.#readClient
+      .connect()
+      .then(() => {
+        // Disposed before the connection finished coming up — close it now.
+        if (this.#disposed) this.#safeDestroyReadClient()
+      })
+      .catch(() => {
+        // Read connection failed to come up; slot reads no-op and back off.
+      })
+
+    // `unref()` so an idle scheduler timer never holds the process
+    // open in single-shot scripts / tests.
+    this.#schedulerTimer = setInterval(() => void this.#runSchedulerTick(), this.#schedulerIntervalMs)
+    this.#schedulerTimer.unref?.()
+  }
+
+  // ── Public API ────────────────────────────────────────────────────
+
+  public async enqueue(input: EnqueueInput): Promise<void> {
+    this.#ensureLive()
+    const stream = this.#streamKey(input.type, input.handlerVersion)
+    // Ensure the consumer group exists eagerly — workers subscribing
+    // later will pick up XADDs that happen between scheduler dispatch
+    // and the worker's group-create call only when the group already
+    // claims the stream's history.
+    await this.#ensureGroup(stream)
+
+    const dueAt = input.notBefore?.getTime()
+    if (dueAt !== undefined && dueAt > Date.now()) {
+      const member = JSON.stringify({
+        taskId: input.taskId,
+        type: input.type,
+        handlerVersion: input.handlerVersion,
+      })
+      await this.#client.zAdd(this.#schedulerKey(), { score: dueAt, value: member })
+      return
+    }
+
+    await this.#client.xAdd(stream, '*', {
+      taskId: input.taskId,
+      type: input.type,
+      handlerVersion: String(input.handlerVersion),
+    })
+  }
+
+  public subscribe(subscription: WorkerSubscription): Disposable {
+    this.#ensureLive()
+
+    const streams = this.#streamsFor(subscription)
+    const ac = new AbortController()
+    const consumers: string[] = []
+    const slotPromises: Array<Promise<void>> = []
+
+    const state: SubscriptionState = {
+      subscription,
+      consumers,
+      streams,
+      abortController: ac,
+      slotPromises,
+    }
+    this.#subscriptions.add(state)
+
+    void (async () => {
+      for (const stream of streams) {
+        await this.#ensureGroup(stream)
+      }
+      for (let i = 0; i < subscription.concurrency; i++) {
+        const consumerName = `${subscription.workerId}-slot-${i}`
+        consumers.push(consumerName)
+        slotPromises.push(this.#runSlot(state, consumerName, ac.signal))
+      }
+    })().catch(() => {
+      // Setup failure leaves the subscription empty; disposal is still
+      // safe, and the caller will see no claims arrive — surfaceable
+      // via telemetry once the runner core wires it through.
+    })
+
+    return {
+      [Symbol.dispose]: () => {
+        ac.abort()
+        this.#subscriptions.delete(state)
+      },
+    }
+  }
+
+  public async heartbeat(claim: ClaimedTask): Promise<void> {
+    if (this.#disposed) return
+    const receipt = claim.receipt as RedisReceipt
+    try {
+      await this.#client.xClaimJustId(receipt.stream, this.#group, receipt.consumer, 0, [receipt.msgId])
+    } catch {
+      // Heartbeat is best-effort. A transient broker error means the
+      // PEL idle counter keeps ticking; XAUTOCLAIM will reclaim the
+      // attempt if the issue persists past the visibility timeout.
+    }
+    if (this.#concurrencyLimits[claim.type] !== undefined) {
+      await this.#refreshCap(claim.type, `${receipt.consumer}:${receipt.msgId}`)
+    }
+  }
+
+  public async acquireIdempotencyLease(input: IdempotencyLeaseInput): Promise<string> {
+    this.#ensureLive()
+    const key = this.#idempotencyKey(input.type, input.key)
+    const set = await this.#client.set(key, input.taskId, {
+      NX: true,
+      EX: this.#idempotencyTtlSec,
+    })
+    if (set === 'OK') return input.taskId
+
+    const existing = await this.#client.get(key)
+    return typeof existing === 'string' && existing.length > 0 ? existing : input.taskId
+  }
+
+  public [Symbol.dispose](): void {
+    if (this.#disposed) return
+    this.#disposed = true
+    clearInterval(this.#schedulerTimer)
+    for (const state of this.#subscriptions) {
+      state.abortController.abort()
+    }
+    this.#subscriptions.clear()
+    this.#safeDestroyReadClient()
+  }
+
+  #safeDestroyReadClient(): void {
+    try {
+      if (this.#readClient.isOpen) this.#readClient.destroy()
+    } catch {
+      // Best-effort: the duplicated connection is force-closed on dispose.
+    }
+  }
+
+  // ── Internal helpers ──────────────────────────────────────────────
+
+  #streamKey(type: string, version: number): string {
+    return `${this.#topicPrefix}tasks:queue:${type}:v${version}`
+  }
+
+  #idempotencyKey(type: string, key: string): string {
+    return `${this.#topicPrefix}tasks:idem:${type}:${key}`
+  }
+
+  #schedulerKey(): string {
+    return `${this.#topicPrefix}tasks:scheduler`
+  }
+
+  /**
+   * Drains due delayed entries from the scheduler ZSET. Re-entrant
+   * guard prevents overlapping ticks on slow brokers; errors are
+   * swallowed so a transient `EVAL` failure does not poison the
+   * timer. The Lua script is the only authority for race-free
+   * dispatch — see {@link SCHEDULER_DISPATCH_SCRIPT}.
+   */
+  async #runSchedulerTick(): Promise<void> {
+    if (this.#disposed || this.#schedulerRunning) return
+    this.#schedulerRunning = true
+    try {
+      await this.#eval(
+        SCHEDULER_DISPATCH_SCRIPT,
+        [this.#schedulerKey()],
+        [String(Date.now()), String(SCHEDULER_BATCH_LIMIT), this.#topicPrefix],
+      )
+    } catch {
+      // Best-effort: a transient broker failure means due entries stay
+      // in the ZSET and the next tick retries.
+    } finally {
+      this.#schedulerRunning = false
+    }
+  }
+
+  /**
+   * Runs a Lua script via the redis client's `eval` (keys/arguments
+   * option shape). The cast keeps {@link RedisLikeClient} opaque to the
+   * exact `redis` command typings.
+   */
+  #eval(script: string, keys: string[], args: string[]): Promise<unknown> {
+    const evalFn = this.#client.eval as (
+      script: string,
+      opts: { keys: string[]; arguments: string[] },
+    ) => Promise<unknown>
+    return evalFn.call(this.#client, script, { keys, arguments: args })
+  }
+
+  // ── Fleet cap ─────────────────────────────────────────────────────
+
+  #capKey(type: string): string {
+    return `${this.#topicPrefix}tasks:cap:${type}`
+  }
+
+  /**
+   * Admit a holder token into the type's fleet-cap lane. Returns `false`
+   * when the lane is full (caller releases the claim back to the stream)
+   * or when the broker errors — failing closed avoids over-admitting.
+   */
+  async #acquireCap(type: string, token: string, cap: number): Promise<boolean> {
+    const now = Date.now()
+    const expiry = now + this.#visibilityFor(type)
+    try {
+      const result = await this.#eval(
+        CAP_ACQUIRE_SCRIPT,
+        [this.#capKey(type)],
+        [String(now), String(cap), token, String(expiry)],
+      )
+      return result === 1
+    } catch {
+      return false
+    }
+  }
+
+  async #releaseCap(type: string, token: string): Promise<void> {
+    try {
+      await this.#client.zRem(this.#capKey(type), token)
+    } catch {
+      // Best-effort: a leaked holder expires via its ZSET score TTL.
+    }
+  }
+
+  async #refreshCap(type: string, token: string): Promise<void> {
+    const expiry = Date.now() + this.#visibilityFor(type)
+    try {
+      await this.#client.zAdd(this.#capKey(type), { score: expiry, value: token }, { XX: true })
+    } catch {
+      // Best-effort: a missed refresh only risks an early reclaim, which
+      // the reclaim path already handles idempotently.
+    }
+  }
+
+  #streamsFor(subscription: WorkerSubscription): string[] {
+    const streams = new Set<string>()
+    for (const type of subscription.types) {
+      const versions = subscription.compatibleVersions[type]
+      const expanded = versions && versions.length > 0 ? Array.from(versions) : [1]
+      for (const version of expanded) {
+        streams.add(this.#streamKey(type, version))
+      }
+    }
+    return Array.from(streams)
+  }
+
+  #visibilityFor(type: string): number {
+    const override = this.#visibilityByType[type]
+    return typeof override === 'number' ? override : this.#defaultVisibilityMs
+  }
+
+  async #ensureGroup(stream: string): Promise<void> {
+    if (this.#ensuredGroups.has(stream)) return
+    try {
+      await this.#client.xGroupCreate(stream, this.#group, '$', { MKSTREAM: true })
+    } catch (error) {
+      // BUSYGROUP = group already exists (idempotent setup).
+      if (!isBusyGroupError(error)) throw error
+    }
+    this.#ensuredGroups.add(stream)
+  }
+
+  async #runSlot(state: SubscriptionState, consumerName: string, signal: AbortSignal): Promise<void> {
+    const { subscription, streams } = state
+
+    // Block until the dedicated read connection is live before issuing any
+    // blocking command on it.
+    await this.#readConnectPromise
+
+    while (!signal.aborted && !this.#disposed) {
+      if (subscription.shouldDrain()) return
+
+      // 1. Try to reclaim any stale PEL entries (broker-side reclaim).
+      const reclaimed = await this.#tryReclaim(streams, consumerName, signal)
+      if (reclaimed) {
+        await this.#deliver(state, reclaimed.stream, reclaimed.entry, consumerName, signal)
+        continue
+      }
+
+      if (signal.aborted || this.#disposed) return
+      if (subscription.shouldDrain()) return
+
+      // 2. Read fresh entries.
+      const fresh = await this.#tryRead(streams, consumerName)
+      if (!fresh) {
+        // Broker error — short backoff to avoid spin.
+        await sleep(this.#retryBackoffMs, signal)
+        continue
+      }
+      if (fresh.entry) {
+        await this.#deliver(state, fresh.stream, fresh.entry, consumerName, signal)
+      }
+    }
+  }
+
+  async #tryReclaim(
+    streams: string[],
+    consumerName: string,
+    signal: AbortSignal,
+  ): Promise<{ stream: string; entry: StreamMessageEntry } | undefined> {
+    for (const stream of streams) {
+      if (signal.aborted || this.#disposed) return undefined
+      const minIdle = this.#visibilityForStream(stream)
+      let reply: XAutoClaimReply | undefined
+      try {
+        reply = await this.#readClient.xAutoClaim(stream, this.#group, consumerName, minIdle, '0', {
+          COUNT: 1,
+        })
+      } catch {
+        return undefined
+      }
+      const entry = pickFirstEntry(reply?.messages)
+      if (entry) return { stream, entry }
+    }
+    return undefined
+  }
+
+  async #tryRead(
+    streams: string[],
+    consumerName: string,
+  ): Promise<{ stream: string; entry: StreamMessageEntry | undefined } | undefined> {
+    let reply: XReadGroupReply | null
+    try {
+      reply = await this.#readClient.xReadGroup(
+        this.#group,
+        consumerName,
+        streams.map((key) => ({ key, id: '>' })),
+        { BLOCK: this.#blockMs, COUNT: 1 },
+      )
+    } catch {
+      return undefined
+    }
+    if (!reply || reply.length === 0) return { stream: streams[0] ?? '', entry: undefined }
+    const first = reply[0]
+    if (!first) return { stream: streams[0] ?? '', entry: undefined }
+    const entry = first.messages[0]
+    return { stream: first.name, entry }
+  }
+
+  async #deliver(
+    state: SubscriptionState,
+    stream: string,
+    entry: StreamMessageEntry,
+    consumerName: string,
+    signal: AbortSignal,
+  ): Promise<void> {
+    const { taskId } = entry.message
+    const { type } = entry.message
+    if (typeof taskId !== 'string' || typeof type !== 'string') {
+      // Malformed entry — drop it. Without a taskId we can't route it,
+      // so leaving it in the PEL would block the consumer forever.
+      try {
+        await this.#client.xAck(stream, this.#group, entry.id)
+      } catch {
+        // Swallow — the autoclaim loop will retry on the next cycle.
+      }
+      return
+    }
+
+    const limit = this.#concurrencyLimits[type]
+    const capToken = `${consumerName}:${entry.id}`
+    if (limit !== undefined) {
+      const acquired = await this.#acquireCap(type, capToken, limit)
+      if (!acquired) {
+        // Lane at fleet cap: release the entry back to the stream tail and
+        // back off so this slot does not hot-loop re-reading it. Ordering
+        // churn is the accepted trade-off of FIFO + fleet cap (PRD §11).
+        try {
+          await this.#client.xAck(stream, this.#group, entry.id)
+          await this.#client.xAdd(stream, '*', entry.message)
+        } catch {
+          // Ack/re-add failed: entry stays in PEL, XAUTOCLAIM recovers it.
+        }
+        await sleep(this.#retryBackoffMs, signal)
+        return
+      }
+    }
+
+    const claim: ClaimedTask = {
+      taskId,
+      type,
+      receipt: {
+        stream,
+        msgId: entry.id,
+        consumer: consumerName,
+        deliveryCount: 0,
+      } satisfies RedisReceipt,
+    }
+
+    try {
+      let outcome: ClaimOutcome
+      try {
+        outcome = await state.subscription.onClaim(claim)
+      } catch {
+        outcome = { kind: 'requeue' }
+      }
+
+      try {
+        if (outcome.kind === 'requeue') {
+          await this.#client.xAck(stream, this.#group, entry.id)
+          await this.#client.xAdd(stream, '*', entry.message)
+        } else {
+          await this.#client.xAck(stream, this.#group, entry.id)
+        }
+      } catch {
+        // Ack failure leaves the entry in PEL; the next XAUTOCLAIM cycle
+        // will surface it for re-delivery once the visibility timeout
+        // expires. Better than silently leaking the message.
+      }
+    } finally {
+      // Release the fleet-cap slot on every outcome — including
+      // `suspended` (awaitChildren frees the worker slot, so it must
+      // free the lane too, else a deep DAG deadlocks its own lane).
+      if (limit !== undefined) await this.#releaseCap(type, capToken)
+    }
+  }
+
+  #visibilityForStream(stream: string): number {
+    // Stream key shape: `${prefix}tasks:queue:${type}:v${version}`.
+    // Slice to recover the type for per-type override lookup.
+    const marker = 'tasks:queue:'
+    const idx = stream.indexOf(marker)
+    if (idx === -1) return this.#defaultVisibilityMs
+    const tail = stream.slice(idx + marker.length)
+    const versionIdx = tail.lastIndexOf(':v')
+    const type = versionIdx === -1 ? tail : tail.slice(0, versionIdx)
+    return this.#visibilityFor(type)
+  }
+
+  #ensureLive(): void {
+    if (this.#disposed) {
+      throw new Error('RedisQueueAdapter has been disposed')
+    }
+  }
+}
+
+const isBusyGroupError = (error: unknown): boolean => error instanceof Error && /BUSYGROUP/i.test(error.message)
+
+const pickFirstEntry = (messages: Array<StreamMessageEntry | null> | undefined): StreamMessageEntry | undefined => {
+  if (!messages) return undefined
+  for (const entry of messages) {
+    if (entry) return entry
+  }
+  return undefined
+}
+
+const sleep = (ms: number, signal: AbortSignal): Promise<void> =>
+  new Promise<void>((resolve) => {
+    if (signal.aborted) {
+      resolve()
+      return
+    }
+    const timer = setTimeout(() => {
+      signal.removeEventListener('abort', onAbort)
+      resolve()
+    }, ms)
+    const onAbort = (): void => {
+      clearTimeout(timer)
+      resolve()
+    }
+    signal.addEventListener('abort', onAbort, { once: true })
+  })