@adonis-agora/durable 0.49.1 → 0.50.0

package/dist/src/transports/queue.js

@@ -1,7 +1,13 @@
 import { randomUUID } from 'node:crypto';
-import { Pollers } from '../pollers.js';
-import { runStepHandler } from '../protocol.js';
+import { atLeastOne, Pollers } from '../pollers.js';
+import { runStepHandler, withTurnHeartbeat } from '../protocol.js';
 import { sanitizeQueueToken, tenantGroup } from '../tenant-group.js';
+import { isWorkflowTask } from '../workflow-turn.js';
+import { AdaptiveController } from './adaptive-concurrency.js';
+import { workerHeartbeatKey } from './bullmq/naming.js';
+import { heartbeatKeyValue, parseHeartbeatValue, WORKER_HEARTBEAT_INTERVAL_MS, WORKER_HEARTBEAT_TTL_SECONDS, } from './bullmq/serialization.js';
+import { DEFAULT_INBOX_SWEEP_INTERVAL_MS, DEFAULT_INBOX_TTL_MS, DEFAULT_TENANT_SUBSCRIBERS_CACHE_MS, InboxRegistry, inboxRegistrationsQueue, runRepliesQueue, tenantEventsQueue, } from './queue-remote.js';
+import { failedTaskIdentity, fanOutDecision, infrastructureFailure } from './task-failure.js';
 import { cachedLiveKeys, descriptorsOn, liveGroupsOf, NO_WORKER_KEYS, workerKeyWriter, } from './worker-keys.js';
 /**
  * The wire payloads carried as a queue job's `payload`. Everything that crosses the queue is plain
@@ -25,16 +31,31 @@ export function fromJson(value) {
 }
 /** How often the engine-side / worker-side poll loops ask the adapter for the next job. */
 const DEFAULT_POLL_INTERVAL_MS = 200;
+/** Idle backoff for the task, heartbeat and control loops: after consecutive empty polls the sleep
+ *  doubles up to this many poll intervals (800ms at the default interval), and the first job found
+ *  resets it. Each handler name has its own task loop, so an idle worker serving many step names
+ *  would otherwise hit the backend `names × 5` times a second. The results loop is exempt — see
+ *  {@link QueueTransportOptions.idleBackoffMax}. */
+const DEFAULT_IDLE_BACKOFF_MAX = 4;
 /** How often (ms) the stalled-job reclaim sweep runs — a coarse, low-frequency check, NOT per poll
  *  tick. Mirrors `DbTransport`'s crash-recovery cadence (its stale-lease default is also 30s). */
 const DEFAULT_STALLED_CHECK_INTERVAL_MS = 30_000;
-/** How old (ms) a claim must be before the sweep presumes its worker dead and re-delivers the job.
- *  Deliberately generous: re-delivery double-runs a step whose worker is merely slow, so this must
- *  comfortably exceed the longest legitimate step execution (see the reclaim doc block for why this
- *  is safe under the durable idempotency contract). The claim's `acquiredAt` is never renewed while
- *  the worker processes — long-running steps hold one claim for their whole duration — so this
- *  default clears real steps in the tens of minutes; a step routinely longer should raise it. */
+/** How old (ms) a claim must be before the sweep presumes its worker dead and re-delivers the job, on
+ *  an adapter that CANNOT renew claims (one without `renewJobs` — every `@adonisjs/queue` driver has it,
+ *  so in practice a custom adapter). There the claim's `acquiredAt` is never
+ *  refreshed while the worker processes — a long step holds one claim for its whole duration — so the
+ *  threshold must comfortably exceed the longest legitimate step: re-delivery double-runs a step whose
+ *  worker is merely slow (safe under the durable idempotency contract, see the reclaim doc block, but
+ *  not free). A step routinely longer than this should raise it. */
 const DEFAULT_STALLED_THRESHOLD_MS = 1_800_000;
+/** The threshold on an adapter that renews claims (`adapter.renewJobs`, on both `@adonisjs/queue` 0.6
+ *  and 0.7). Every
+ *  in-flight claim is renewed every half threshold (see {@link QueueTransport.#renewOpenClaims}), so a
+ *  claim's age measures how long its worker has been SILENT, not how long the step has run: a dead
+ *  worker's job is re-delivered within about this long, and a healthy long step never is. Larger than
+ *  `@boringnode/queue`'s own 30s worker default so a step that blocks the event loop for a while (a
+ *  CPU-heavy body starves the renewal timer) is not mistaken for a dead worker. */
+const DEFAULT_RENEWING_STALLED_THRESHOLD_MS = 120_000;
 /** How many times a single job may be reclaimed before the adapter fails it permanently instead of
  *  re-delivering — bounds a poison job that stalls every worker it touches. */
 const DEFAULT_MAX_STALLED_COUNT = 3;
@@ -87,6 +108,12 @@ export function toBrokerPriority(priority) {
  * is single-consumer (point-to-point), so it is correct for a single engine instance but does NOT
  * fan out to every pod the way a real pub/sub does.
  *
+ * It also carries remote workflow turns (`dispatchWorkflowTask` / `handleWorkflow` / `onDecision`,
+ * streamed step events) and the store-less tenant-pod protocol (start-run, run-request, run replies,
+ * tenant events), so a `tenant` pod can reach a `control-plane` over `@adonisjs/queue` alone. Run
+ * replies and tenant events, which BullMQ broadcasts on Redis PUB/SUB, ride per-instance inbox queues
+ * here — see `queue-remote.ts`.
+ *
  * Run one instance engine-side (`onResult` + `dispatch`) and one per worker process (`handle()`).
  * The wire payloads are the documented `RemoteTask`/`StepResult` JSON, so non-Node workers on the
  * same queues interoperate.
@@ -104,6 +131,26 @@ export class QueueTransport {
     #namespace;
     #explicitNamespace;
     #pollIntervalMs;
+    #closeTimeoutMs;
+    /** Fixed or adaptive task concurrency, plus the in-flight/latency/RSS/CPU telemetry the liveness
+     *  heartbeat carries. One controller for every task loop of this instance, as on BullMQ. */
+    #controller;
+    /** Task loops parked on "every slot busy", woken when an adaptive move raises the limit. */
+    #limitWaiters = new Set();
+    /** The task queues this instance consumes — the reclaim sweep bridges a poison job only on these. */
+    #taskQueues = new Set();
+    /** Routing tokens this instance beats a liveness key for (one per handled step name). */
+    #heartbeatTokens = new Set();
+    #workerHeartbeatIntervalMs;
+    #workerHeartbeatTimer;
+    /** Liveness-key writes still in flight — `close()` lets them land before withdrawing the keys. */
+    #livenessWrites = new Set();
+    /** Routing tokens whose liveness-key write is still in flight (see `#startWorkerHeartbeat`). */
+    #beatsInFlight = new Set();
+    #resultConcurrency;
+    #idleBackoffMax;
+    /** Every job handler still running (any queue) — `close()` waits for these, bounded. */
+    #runningJobs = new Set();
     #stalledThresholdMs;
     #maxStalledCount;
     #instanceId;
@@ -124,13 +171,54 @@ export class QueueTransport {
      *  (`stalledCheckIntervalMs <= 0`) or the adapter can't `recoverStalledJobs`. Separate from
      *  {@link #pollers} because the sweep cadence is unrelated to the fast job-poll cadence. */
     #reclaimPollers;
+    /** Renews the claims in {@link #openJobs} every half threshold. Absent when the adapter can't
+     *  `renewJobs`. */
+    #renewTimer;
+    /** The renewal round in flight, so a slow adapter never stacks rounds and close() can await it. */
+    #renewal;
     /** When set (see {@link deferConsumers}), {@link QueueTransport.#startLoop} parks its loop here
      *  instead of polling — {@link startConsumers} flushes the parked starts. Dispatching (`pushOn`)
      *  is never deferred; only consumption is. */
     #consumersDeferred = false;
     #deferredStarts = [];
     #workerKeys;
+    /** Whether a real key store was configured — without one there is no liveness to report. */
+    #hasWorkerKeys;
     #liveWorkerKeys;
+    /** Workflow turn consumers by workflow name — a `workflow`-shaped task job routes here. */
+    #workflowTurns = new Map();
+    /** Every `onDecision` handler (an engine subscribes one; it applies a decision by run id). */
+    #decisionHandlers = new Set();
+    #decisionsLoop = false;
+    #stepEventsLoop = false;
+    #startRunLoop = false;
+    #runRequestLoop = false;
+    #runRepliesLoop = false;
+    #runReplyHandler;
+    #tenantEventsLoop = false;
+    #registrationsLoop = false;
+    #tenantRole = false;
+    /** Control-plane side: correlation id (requestId, or a start-run's runId) → the requester's instance
+     *  id, held while the request's handler runs so {@link publishRunReply} knows whose inbox to target. */
+    #replyRoutes = new Map();
+    #tenantEventHandlers = new Map();
+    /** Serializes {@link publishTenantEvent} so events reach every inbox in the order they were published. */
+    #tenantPublishChain = Promise.resolve();
+    #inboxes;
+    /**
+     * Per-group worker health, for `engine.workerHealth()` and the dashboard/Telescope worker panels.
+     * Present only when a {@link QueueTransportOptions.workerKeys} store is configured (the
+     * `transports.queue()` default) — without one no worker can advertise liveness, and reporting
+     * "backlog, zero workers" for every group would be a false alarm.
+     *
+     * - `depth` is the task queue's PENDING count (`adapter.sizeOf`), the jobs waiting for a worker.
+     *   Unlike BullMQ's waiting+active+delayed+prioritized it leaves out jobs in flight and delayed
+     *   retries: the `@adonisjs/queue` adapter contract has no count for those. Priorities are included
+     *   (they are pending jobs). An adapter without `sizeOf` reports `0`.
+     * - `liveWorkers` are the unexpired `${P}-worker-heartbeat:<group>:<instance>` keys, each with the
+     *   worker's {@link WorkerStatus} when it carries one (this transport's own beats do).
+     */
+    groupHealth;
     constructor(options) {
         // Typed through CompatAdapter so the same code drives the 0.6 (job-id) and 0.7 (lease) contracts.
         this.#adapter = options.adapter();
@@ -139,7 +227,25 @@ export class QueueTransport {
         this.#namespace = options.namespace;
         this.#explicitNamespace = options.namespace !== undefined;
         this.#pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
-        this.#stalledThresholdMs = options.stalledThresholdMs ?? DEFAULT_STALLED_THRESHOLD_MS;
+        this.#closeTimeoutMs =
+            options.closeTimeoutMs !== undefined &&
+                Number.isFinite(options.closeTimeoutMs) &&
+                options.closeTimeoutMs >= 0
+                ? options.closeTimeoutMs
+                : 5_000;
+        this.#controller = new AdaptiveController({
+            ...(options.concurrency !== undefined ? { concurrency: options.concurrency } : {}),
+            apply: () => {
+                for (const wake of this.#limitWaiters)
+                    wake();
+            },
+        });
+        this.#resultConcurrency = atLeastOne(options.resultConcurrency, 1);
+        this.#idleBackoffMax = atLeastOne(options.idleBackoffMax, DEFAULT_IDLE_BACKOFF_MAX);
+        const canRenew = typeof this.#adapter.renewJobs === 'function';
+        this.#stalledThresholdMs =
+            options.stalledThresholdMs ??
+                (canRenew ? DEFAULT_RENEWING_STALLED_THRESHOLD_MS : DEFAULT_STALLED_THRESHOLD_MS);
         this.#maxStalledCount = options.maxStalledCount ?? DEFAULT_MAX_STALLED_COUNT;
         this.#instanceId = options.instanceId ?? randomUUID();
         this.#onError = options.onError ?? DEFAULT_LOG;
@@ -147,13 +253,44 @@ export class QueueTransport {
         this.#adapter.setWorkerId(this.#instanceId);
         this.#pollers = new Pollers(this.#pollIntervalMs, this.#onError);
         this.#reclaimPollers = this.#startReclaimSweep(options.stalledCheckIntervalMs ?? DEFAULT_STALLED_CHECK_INTERVAL_MS);
+        if (canRenew)
+            this.#startClaimRenewal();
+        this.#hasWorkerKeys = options.workerKeys !== undefined;
+        if (this.#hasWorkerKeys)
+            this.groupHealth = (group) => this.#groupHealth(group);
+        this.#workerHeartbeatIntervalMs = atLeastOne(options.workerHeartbeatIntervalMs, WORKER_HEARTBEAT_INTERVAL_MS);
         this.#workerKeys = options.workerKeys ?? NO_WORKER_KEYS;
         this.#liveWorkerKeys = cachedLiveKeys(this.#workerKeys, options.workerKeysCacheMs ?? 5_000, Date.now);
+        this.#inboxes = new InboxRegistry({
+            store: this.#workerKeys,
+            prefix: () => this.#effectivePrefix(),
+            instanceId: this.#instanceId,
+            ttlMs: atLeastOne(options.inboxTtlMs, DEFAULT_INBOX_TTL_MS),
+            subscribersCacheMs: options.tenantSubscribersCacheMs ?? DEFAULT_TENANT_SUBSCRIBERS_CACHE_MS,
+            sweepIntervalMs: options.inboxSweepIntervalMs ?? DEFAULT_INBOX_SWEEP_INTERVAL_MS,
+            announce: (registration) => this.#adapter
+                .pushOn(inboxRegistrationsQueue(this.#effectivePrefix()), this.#job('inboxRegistration', registration))
+                .then(() => undefined),
+            drain: (queue, max) => this.#drainQueue(queue, max),
+            onError: this.#onError,
+        });
     }
-    /** The writer a worker runtime's registry advertises its descriptor and heartbeat through. */
+    /** The writer a worker runtime's registry advertises its descriptor and heartbeat through: the key
+     *  store when this instance has one. A store-less tenant pod (see {@link useTenantRole}) with no key
+     *  store forwards it to the control plane instead, which records it in its own — so the control plane
+     *  still sees the pod's workers. Otherwise, with no store, nothing is advertised. */
     createDescriptorRedis() {
+        if (this.#workerKeys === NO_WORKER_KEYS && this.#tenantRole) {
+            return this.#inboxes.remoteKeyWriter();
+        }
         return workerKeyWriter(this.#workerKeys, Date.now);
     }
+    /** Declare that this transport backs a store-less `tenant` pod (the provider calls it): worker
+     *  advertisement is then forwarded to the control plane when this instance has no key store. Only a
+     *  tenant pod is guaranteed a control plane that records the forwarded keys. */
+    useTenantRole() {
+        this.#tenantRole = true;
+    }
     /** The live worker descriptors advertised on `token`. */
     async listWorkerDescriptors(token) {
         return descriptorsOn(await this.#liveWorkerKeys(), token);
@@ -229,6 +366,18 @@ export class QueueTransport {
     #controlQueue() {
         return `${this.#effectivePrefix()}:control`;
     }
+    #decisionsQueue() {
+        return `${this.#effectivePrefix()}:decisions`;
+    }
+    #stepEventsQueue() {
+        return `${this.#effectivePrefix()}:step-events`;
+    }
+    #startRunQueue() {
+        return `${this.#effectivePrefix()}:start-run`;
+    }
+    #runRequestQueue() {
+        return `${this.#effectivePrefix()}:run-request`;
+    }
     /**
      * Build the adapter `JobData` envelope for a JSON-safe payload. A translated `priority` is added
      * only when set, so the default FIFO path is untouched for unprioritised dispatches.
@@ -261,14 +410,167 @@ export class QueueTransport {
      */
     handle(name, fn) {
         this.#handlers.set(name, fn);
-        const token = tenantGroup(sanitizeQueueToken(name), this.#partition);
+        this.#ensureTaskLoop(tenantGroup(sanitizeQueueToken(name), this.#partition));
+    }
+    /**
+     * Register a WORKFLOW turn consumer (worker side). Shares the step task loop of the workflow name's
+     * routing token — the same queue {@link dispatchWorkflowTask} targets — and tells the two job kinds
+     * apart BY SHAPE: a `workflow`-shaped job runs `turn(task)` and its {@link WorkflowDecision} is pushed
+     * onto `${P}:decisions`. Re-registering swaps the turn fn.
+     */
+    handleWorkflow(name, turn) {
+        this.#workflowTurns.set(name, turn);
+        this.#ensureTaskLoop(tenantGroup(sanitizeQueueToken(name), this.#partition));
+    }
+    /** Start (once per routing token) the task loop shared by {@link handle} and {@link handleWorkflow}. */
+    #ensureTaskLoop(token) {
         if (this.#taskLoops.has(token))
             return;
-        this.#taskLoops.set(token, this.#startLoop(this.#tasksQueue(token), async (job) => {
-            const task = fromJson(job.payload);
-            const result = await runStepHandler(task, this.#handlers.get(task.name), (beat) => this.heartbeat(beat));
-            await this.#adapter.pushOn(this.#resultsQueue(), this.#job('result', result));
+        const queue = this.#tasksQueue(token);
+        this.#taskQueues.add(queue);
+        const controller = this.#controller;
+        this.#taskLoops.set(token, this.#startLoop(queue, async (job) => {
+            const payload = fromJson(job.payload);
+            const startedAt = Date.now();
+            controller.onStart();
+            // Only step durations feed the latency gradient; a workflow turn still counts as in-flight
+            // work and as forward progress for stall detection (as on bullmq).
+            if (isWorkflowTask(payload)) {
+                try {
+                    await this.#runWorkflowTask(payload);
+                }
+                finally {
+                    controller.onSettle(Date.now() - startedAt, true, 'workflow');
+                }
+                return;
+            }
+            const task = payload;
+            let ok = false;
+            try {
+                const result = await runStepHandler(task, this.#handlers.get(task.name), (beat) => this.heartbeat(beat));
+                await this.#adapter.pushOn(this.#resultsQueue(), this.#job('result', result));
+                ok = result.status === 'completed';
+            }
+            finally {
+                controller.onSettle(Date.now() - startedAt, ok, 'step');
+            }
+        }, {
+            // Adaptive: every tick reads the controller's live limit. Fixed: the plain number, so the
+            // one-slot loop keeps its historical inline shape.
+            concurrency: controller.isAdaptive ? () => controller.limit : controller.limit,
+            idleBackoffMax: this.#idleBackoffMax,
         }));
+        // Advertise liveness only once the loop really consumes: a deferred (producer-only) process must
+        // not look like a live worker.
+        if (this.#consumersDeferred)
+            this.#deferredStarts.push(() => this.#beginServing(token));
+        else
+            this.#beginServing(token);
+    }
+    /** A task loop for `token` is consuming: run the concurrency controller and beat its liveness key. */
+    #beginServing(token) {
+        this.#controller.start();
+        this.#startWorkerHeartbeat(token);
+    }
+    // ---------------------------------------------------------------------------
+    // worker liveness + health — `${P}-worker-heartbeat:<token>:<instanceId>` in the worker-key store
+    // ---------------------------------------------------------------------------
+    /**
+     * Refresh every served routing token's liveness key on one shared interval (default 10s, live 35s),
+     * stamped with the controller's {@link WorkerStatus} snapshot (concurrency mode and live limit,
+     * in-flight, RSS/CPU, throughput, p95, last adjustment) — the same `{ ts, status }` value and key
+     * the BullMQ transport writes, so the dashboard's worker cards read both alike. A new token beats
+     * at once. Best-effort: a failed write goes to `onError` and the key simply expires. A no-op without
+     * a configured {@link QueueTransportOptions.workerKeys} store.
+     */
+    #startWorkerHeartbeat(token) {
+        if (!this.#hasWorkerKeys)
+            return;
+        const ttlMs = Math.max(WORKER_HEARTBEAT_TTL_SECONDS * 1000, this.#workerHeartbeatIntervalMs * 3);
+        const beatOne = (t) => {
+            // One write per token at a time: a beat due while the last one is still in flight is skipped.
+            // Beats are not queued behind a slow store — every interval used to start a write regardless,
+            // so a store slower than the interval piled up writes without bound, each holding a pool
+            // connection the reads (groupHealth, the claim loop) then waited behind.
+            if (this.#beatsInFlight.has(t))
+                return;
+            this.#beatsInFlight.add(t);
+            const now = Date.now();
+            const key = workerHeartbeatKey(this.#effectivePrefix(), t, this.#instanceId);
+            const value = heartbeatKeyValue(now, this.#controller.snapshot());
+            const write = Promise.resolve()
+                .then(() => this.#workerKeys.put(key, value, now + ttlMs))
+                .catch((err) => this.#onError(err))
+                .finally(() => this.#beatsInFlight.delete(t));
+            this.#livenessWrites.add(write);
+            void write.finally(() => this.#livenessWrites.delete(write));
+        };
+        const isNew = !this.#heartbeatTokens.has(token);
+        this.#heartbeatTokens.add(token);
+        if (isNew)
+            beatOne(token);
+        if (this.#workerHeartbeatTimer)
+            return;
+        this.#workerHeartbeatTimer = setInterval(() => {
+            for (const t of this.#heartbeatTokens)
+                beatOne(t);
+        }, this.#workerHeartbeatIntervalMs);
+        this.#workerHeartbeatTimer.unref?.();
+    }
+    async #groupHealth(group) {
+        const [depth, entries] = await Promise.all([
+            this.#adapter.sizeOf ? this.#adapter.sizeOf(this.#tasksQueue(group)) : Promise.resolve(0),
+            this.#liveWorkerKeys(),
+        ]);
+        const marker = workerHeartbeatKey(this.#effectivePrefix(), group, '');
+        const liveWorkers = [];
+        for (const { key, value } of entries) {
+            if (!key.startsWith(marker))
+                continue;
+            const parsed = parseHeartbeatValue(value);
+            liveWorkers.push({
+                group,
+                instanceId: key.slice(marker.length),
+                lastBeatAt: parsed.lastBeatAt,
+                // Passed through as-is (worker-authored); a worker runtime's compact `{ ts, status: 'up' }`
+                // beat carries no status object and reads as liveness only.
+                ...(parsed.status !== undefined
+                    ? { status: parsed.status }
+                    : {}),
+            });
+        }
+        return { group, depth: Number(depth) || 0, liveWorkers };
+    }
+    /** Replay one workflow turn and push its decision. An unknown workflow or a throwing turn still
+     *  yields a `failed` decision so the run settles; a failed push throws, so the job is redelivered. */
+    async #runWorkflowTask(task) {
+        let decision;
+        const turn = this.#workflowTurns.get(task.workflow);
+        try {
+            decision = turn
+                ? await withTurnHeartbeat(task, (beat) => this.heartbeat(beat), () => turn(task))
+                : {
+                    taskId: task.taskId,
+                    runId: task.runId,
+                    status: 'failed',
+                    commands: [],
+                    error: {
+                        message: `no workflow registered for '${task.workflow}'`,
+                        code: 'no_workflow',
+                    },
+                };
+        }
+        catch (err) {
+            this.#onError(err);
+            decision = {
+                taskId: task.taskId,
+                runId: task.runId,
+                status: 'failed',
+                commands: [],
+                error: { message: err instanceof Error ? err.message : String(err) },
+            };
+        }
+        await this.#adapter.pushOn(this.#decisionsQueue(), this.#job('decision', decision));
     }
     /** Worker side: publish a liveness heartbeat for an in-flight long step. */
     async heartbeat(beat) {
@@ -280,12 +582,12 @@ export class QueueTransport {
     onResult(handler) {
         this.#startLoop(this.#resultsQueue(), async (job) => {
             await handler(fromJson(job.payload));
-        });
+        }, { concurrency: this.#resultConcurrency, idleBackoffMax: 1 });
     }
     onHeartbeat(handler) {
         this.#startLoop(this.#heartbeatsQueue(), async (job) => {
             await handler(fromJson(job.payload));
-        });
+        }, { idleBackoffMax: this.#idleBackoffMax });
     }
     // ---------------------------------------------------------------------------
     // control plane (best-effort, point-to-point)
@@ -297,7 +599,243 @@ export class QueueTransport {
     onControl(handler) {
         this.#startLoop(this.#controlQueue(), async (job) => {
             handler(fromJson(job.payload));
+        }, { idleBackoffMax: this.#idleBackoffMax });
+    }
+    // ---------------------------------------------------------------------------
+    // remote workflows — engine ⇄ workflow worker (point-to-point, like the step queues)
+    // ---------------------------------------------------------------------------
+    /** engine → workflow worker: a {@link WorkflowTask} on the workflow's routing-token task queue (job
+     *  `workflow`), the queue a {@link handleWorkflow} consumer polls. */
+    async dispatchWorkflowTask(task) {
+        await this.#adapter.pushOn(this.#tasksQueue(task.group), this.#job('workflow', task, task.priority));
+    }
+    /**
+     * engine ← workflow worker: consume {@link WorkflowDecision}s from `${P}:decisions`. ONE loop per
+     * transport; every registered handler sees every decision (the engine applies one by run id and drops
+     * any its run no longer awaits), so several remote workflows can share the transport.
+     */
+    onDecision(handler) {
+        this.#decisionHandlers.add(handler);
+        if (this.#decisionsLoop)
+            return;
+        this.#decisionsLoop = true;
+        this.#startLoop(this.#decisionsQueue(), async (job) => {
+            const decision = fromJson(job.payload);
+            // Every handler, concurrently; all settle before a failure redelivers the decision.
+            await fanOutDecision(this.#decisionHandlers, decision, this.#onError);
+        }, { idleBackoffMax: 1 });
+    }
+    /** workflow worker → engine: a local step's lifecycle on `${P}:step-events` (job `stepEvent`),
+     *  point-to-point so ONE engine instance checkpoints each event once. */
+    async dispatchStepEvent(event) {
+        await this.#adapter.pushOn(this.#stepEventsQueue(), this.#job('stepEvent', event));
+    }
+    /** engine ← workflow worker: consume streamed step events. Idempotent — a second call is a no-op. */
+    onStepEvent(handler) {
+        if (this.#stepEventsLoop)
+            return;
+        this.#stepEventsLoop = true;
+        this.#startLoop(this.#stepEventsQueue(), async (job) => {
+            await handler(fromJson(job.payload));
+        }, { idleBackoffMax: this.#idleBackoffMax });
+    }
+    // ---------------------------------------------------------------------------
+    // store-less protocol (P4) — tenant pod ⇄ control plane
+    //
+    // start-run and run-request are point-to-point queues consumed by the control plane, exactly as on
+    // BullMQ. Run replies and tenant events — Redis PUB/SUB on BullMQ — go to per-instance inbox queues
+    // instead, since a queue job has one consumer: a reply to the requester's inbox (its id rides the
+    // request as `replyTo`), a tenant event to the inbox of each live subscriber of that tenant (read from
+    // the worker-key store). See `queue-remote.ts` for the design and its limits.
+    // ---------------------------------------------------------------------------
+    /** tenant → control plane: enqueue a {@link StartRunMessage} on `${P}:start-run` (job `startRun`),
+     *  stamped with this instance as the reply target. */
+    async dispatchStartRun(msg) {
+        this.#ensureRunRepliesLoop();
+        const payload = { ...msg, replyTo: this.#instanceId };
+        await this.#adapter.pushOn(this.#startRunQueue(), this.#job('startRun', payload));
+    }
+    /** control plane ← tenant: consume {@link StartRunMessage}s. Idempotent — a second call is a no-op. */
+    onStartRun(handler) {
+        this.#ensureRegistrationsLoop();
+        if (this.#startRunLoop)
+            return;
+        this.#startRunLoop = true;
+        this.#startLoop(this.#startRunQueue(), async (job) => {
+            const { replyTo, ...msg } = fromJson(job.payload);
+            await this.#withReplyRoute(msg.runId, replyTo, () => handler(msg));
+        }, { idleBackoffMax: this.#idleBackoffMax });
+    }
+    /** tenant → control plane: enqueue a {@link RunRequest} on `${P}:run-request` (job `runRequest`),
+     *  stamped with this instance as the reply target. */
+    async dispatchRunRequest(msg) {
+        this.#ensureRunRepliesLoop();
+        const payload = { ...msg, replyTo: this.#instanceId };
+        await this.#adapter.pushOn(this.#runRequestQueue(), this.#job('runRequest', payload));
+    }
+    /** control plane ← tenant: consume {@link RunRequest}s. Idempotent — a second call is a no-op. */
+    onRunRequest(handler) {
+        this.#ensureRegistrationsLoop();
+        if (this.#runRequestLoop)
+            return;
+        this.#runRequestLoop = true;
+        this.#startLoop(this.#runRequestQueue(), async (job) => {
+            const { replyTo, ...msg } = fromJson(job.payload);
+            await this.#withReplyRoute(msg.requestId, replyTo, () => handler(msg));
+        }, { idleBackoffMax: this.#idleBackoffMax });
+    }
+    /** Remember who asked for `correlationId` while `work` (the responder, which replies inside it) runs. */
+    async #withReplyRoute(correlationId, replyTo, work) {
+        if (correlationId === undefined || replyTo === undefined)
+            return work();
+        this.#replyRoutes.set(correlationId, replyTo);
+        try {
+            await work();
+        }
+        finally {
+            if (this.#replyRoutes.get(correlationId) === replyTo)
+                this.#replyRoutes.delete(correlationId);
+        }
+    }
+    /**
+     * control plane → tenant: push a correlated {@link RunReply} onto the inbox of the instance that sent
+     * the request (`${P}:run-replies:<instance>`). Only the instance that consumed the request knows the
+     * requester, so this must run while that request is being handled — which is what the responder does.
+     * A reply with no known requester (a request that carried no `replyTo`) is dropped and reported.
+     */
+    async publishRunReply(reply) {
+        const owner = this.#replyRoutes.get(reply.requestId);
+        if (owner === undefined) {
+            this.#onError(new Error(`[QueueTransport] dropped the run reply for "${reply.requestId}": no requester is known for it on this instance`));
+            return;
+        }
+        const queue = runRepliesQueue(this.#effectivePrefix(), owner);
+        await this.#adapter.pushOn(queue, this.#job('runReply', reply));
+        this.#inboxes.notePush(queue, owner);
+    }
+    /**
+     * tenant ← control plane: consume this instance's reply inbox. Idempotent — a second call is a no-op
+     * (one gateway per transport matches every reply by id). The inbox is private to this instance, so it
+     * is consumed even when consumers are deferred: no other process could take its jobs.
+     */
+    onRunReply(handler) {
+        if (this.#runReplyHandler)
+            return;
+        this.#runReplyHandler = handler;
+        this.#ensureRunRepliesLoop();
+    }
+    /** Start consuming the reply inbox (once). Also started on the first request, so a reply is never
+     *  left for a dead queue just because the gateway subscribed late. */
+    #ensureRunRepliesLoop() {
+        if (this.#runRepliesLoop)
+            return;
+        this.#runRepliesLoop = true;
+        this.#inboxes.activateInbox();
+        // On the critical path of every proxied call: never back off (like the results loop).
+        this.#reallyStartLoop(runRepliesQueue(this.#effectivePrefix(), this.#instanceId), async (job) => {
+            const reply = fromJson(job.payload);
+            try {
+                this.#runReplyHandler?.(reply);
+            }
+            catch (err) {
+                this.#onError(err); // a throwing listener must not redeliver the reply
+            }
+        }, { idleBackoffMax: 1 });
+    }
+    /**
+     * control plane → tenant: push a {@link TenantEvent} onto the inbox of every instance with a live
+     * subscription to `evt.tenant` (none: nothing is pushed). Publishes are serialized, so each
+     * subscriber receives one publisher's events in publish order.
+     */
+    publishTenantEvent(evt) {
+        this.#ensureRegistrationsLoop();
+        const run = this.#tenantPublishChain.then(async () => {
+            const owners = await this.#inboxes.subscribersOf(evt.tenant);
+            for (const owner of owners) {
+                const queue = tenantEventsQueue(this.#effectivePrefix(), owner);
+                await this.#adapter.pushOn(queue, this.#job('tenantEvent', evt));
+                this.#inboxes.notePush(queue, owner);
+            }
         });
+        this.#tenantPublishChain = run.catch(() => undefined);
+        return run;
+    }
+    /**
+     * control plane: record the inbox registrations instances announce (their subscriptions and inbox
+     * liveness) in this instance's worker-key store, where {@link publishTenantEvent} and the janitor read
+     * them. Started by whichever control-plane surface comes up first (the responder's consumers, or the
+     * first publish). Point-to-point: control-plane replicas share the store, so one recording suffices.
+     */
+    #ensureRegistrationsLoop() {
+        if (this.#registrationsLoop)
+            return;
+        this.#registrationsLoop = true;
+        this.#startLoop(inboxRegistrationsQueue(this.#effectivePrefix()), (job) => this.#inboxes.record(fromJson(job.payload)), { idleBackoffMax: this.#idleBackoffMax });
+    }
+    /**
+     * tenant ← control plane: subscribe to `tenant`'s events. Announces the subscription to the control
+     * plane (re-announced every third of {@link QueueTransportOptions.inboxTtlMs} while it lasts) and
+     * consumes this instance's event inbox. Needs no store on this side. Returns an unsubscribe fn that
+     * detaches only this handler; the last one withdraws the subscription.
+     */
+    onTenantEvent(tenant, handler) {
+        let handlers = this.#tenantEventHandlers.get(tenant);
+        if (!handlers) {
+            handlers = new Set();
+            this.#tenantEventHandlers.set(tenant, handlers);
+        }
+        handlers.add(handler);
+        this.#inboxes.subscribe(tenant);
+        this.#ensureTenantEventsLoop();
+        return () => {
+            const set = this.#tenantEventHandlers.get(tenant);
+            if (!set?.delete(handler) || set.size > 0)
+                return;
+            this.#tenantEventHandlers.delete(tenant);
+            this.#inboxes.unsubscribe(tenant);
+        };
+    }
+    /** Consume the tenant-event inbox (once; private to this instance, so never deferred). Keeps running
+     *  after the last unsubscribe so events pushed before the withdrawal are drained, not left behind. */
+    #ensureTenantEventsLoop() {
+        if (this.#tenantEventsLoop)
+            return;
+        this.#tenantEventsLoop = true;
+        this.#reallyStartLoop(tenantEventsQueue(this.#effectivePrefix(), this.#instanceId), async (job) => {
+            const evt = fromJson(job.payload);
+            for (const h of this.#tenantEventHandlers.get(evt.tenant) ?? []) {
+                try {
+                    h(evt);
+                }
+                catch (err) {
+                    this.#onError(err); // a throwing listener must not redeliver (and reorder) the event
+                }
+            }
+        }, { idleBackoffMax: this.#idleBackoffMax });
+    }
+    /** Withdraw this instance's advertisements, then drop what is left in its own inboxes: replies and
+     *  events addressed to a process that is going away have no one else to go to. */
+    async #closeInboxes() {
+        await this.#inboxes.close();
+        const prefix = this.#effectivePrefix();
+        const own = [
+            ...(this.#runRepliesLoop ? [runRepliesQueue(prefix, this.#instanceId)] : []),
+            ...(this.#tenantEventsLoop ? [tenantEventsQueue(prefix, this.#instanceId)] : []),
+        ];
+        for (const queue of own) {
+            await this.#drainQueue(queue, 1_000).catch((err) => this.#onError(err));
+        }
+    }
+    /** Pop and finalize up to `max` jobs of `queue` without running them (the janitor, and close()).
+     *  Resolves `true` once the queue came back empty. */
+    async #drainQueue(queue, max) {
+        for (let i = 0; i < max; i += 1) {
+            const job = await this.#adapter.popFrom(queue);
+            if (!job)
+                return true;
+            await this.#adapter.completeJob(jobHandle(job), queue);
+        }
+        return false;
     }
     // ---------------------------------------------------------------------------
     // poll loop
@@ -317,7 +855,7 @@ export class QueueTransport {
      * reconcile recovery instead of wedging or losing the result.) A job nobody can settle loops at
      * the poll rate and says so on every attempt, which is the failure we want — loud, not invisible.
      */
-    #startLoop(queue, onJob) {
+    #startLoop(queue, onJob, opts = {}) {
         // Deferred consumption (see deferConsumers): park the start; startConsumers() flushes it. The
         // handle returned now still stops the loop whether it is flushed later or never.
         if (this.#consumersDeferred) {
@@ -325,7 +863,7 @@ export class QueueTransport {
             let live;
             this.#deferredStarts.push(() => {
                 if (!cancelled)
-                    live = this.#reallyStartLoop(queue, onJob);
+                    live = this.#reallyStartLoop(queue, onJob, opts);
             });
             return {
                 stop: () => {
@@ -334,12 +872,23 @@ export class QueueTransport {
                 },
             };
         }
-        return this.#reallyStartLoop(queue, onJob);
+        return this.#reallyStartLoop(queue, onJob, opts);
     }
-    #reallyStartLoop(queue, onJob) {
+    #reallyStartLoop(queue, onJob, { concurrency = 1, idleBackoffMax }) {
         // Record every queue we consume from so the reclaim sweep can walk exactly this set.
         this.#poppedQueues.add(queue);
+        // This loop's jobs still in a handler. With one fixed slot the tick awaits its job (the historical
+        // shape); otherwise it hands each job off and keeps popping until every slot is taken, so N
+        // slots cost ONE poll loop, not N. A live (adaptive) limit is re-read on every tick.
+        const limit = typeof concurrency === 'function' ? concurrency : () => concurrency;
+        const inFlight = new Set();
         return this.#pollers.start(async () => {
+            if (inFlight.size >= limit()) {
+                // Every slot busy: wait for one to free (or the limit to move), then report work so the
+                // loop re-ticks at once.
+                await this.#slotFreed(inFlight);
+                return true;
+            }
             let job;
             try {
                 job = await this.#adapter.popFrom(queue);
@@ -350,11 +899,42 @@ export class QueueTransport {
             }
             if (!job)
                 return false;
-            // Track the popped-but-unfinished job so close() can hand it back to the broker instead of
-            // destroying the adapter under it (which stranded it `active` until the stalled sweep —
-            // every deploy of a busy fleet parked its mid-flight jobs for up to the stalled threshold).
-            const handle = jobHandle(job);
-            this.#openJobs.set(job.id, { queue, handle });
+            const run = this.#runJob(queue, job, onJob);
+            if (concurrency === 1) {
+                await run;
+                return true;
+            }
+            inFlight.add(run);
+            void run.finally(() => inFlight.delete(run));
+            return true;
+        }, idleBackoffMax === undefined ? undefined : { idleBackoffMax });
+    }
+    /** Resolve when one of `inFlight` settles or the adaptive controller moves the limit. */
+    async #slotFreed(inFlight) {
+        let wake = () => { };
+        const moved = new Promise((resolve) => {
+            wake = resolve;
+        });
+        this.#limitWaiters.add(wake);
+        try {
+            await Promise.race([...inFlight, moved]);
+        }
+        finally {
+            this.#limitWaiters.delete(wake);
+        }
+    }
+    /**
+     * Run one popped job and finalize it: complete on success, hand back for redelivery on a throw.
+     * Tracked in {@link #openJobs} (renewed, and handed back by `close()` if unfinished) and in
+     * {@link #runningJobs} (waited for by `close()`, bounded).
+     */
+    #runJob(queue, job, onJob) {
+        // Track the popped-but-unfinished job so close() can hand it back to the broker instead of
+        // destroying the adapter under it (which stranded it `active` until the stalled sweep —
+        // every deploy of a busy fleet parked its mid-flight jobs for up to the stalled threshold).
+        const handle = jobHandle(job);
+        this.#openJobs.set(job.id, { queue, handle });
+        const run = (async () => {
             try {
                 await onJob(job);
                 // A lease-based adapter answers `false` when the lease was lost (the reclaim sweep re-delivered
@@ -372,8 +952,10 @@ export class QueueTransport {
             finally {
                 this.#openJobs.delete(job.id);
             }
-            return true;
-        });
+        })();
+        this.#runningJobs.add(run);
+        void run.finally(() => this.#runningJobs.delete(run));
+        return run;
     }
     // ---------------------------------------------------------------------------
     // stalled-job reclaim
@@ -395,8 +977,9 @@ export class QueueTransport {
      * engine ignores results for a checkpoint that already completed — so re-running a reclaimed step, or
      * re-delivering the two production shapes (result published but `completeJob` never reached; or claimed
      * and died before any result), cannot corrupt a run. The only cost is wasted work when a merely-slow
-     * worker's job is re-delivered, which is why {@link DEFAULT_STALLED_THRESHOLD_MS} is deliberately
-     * generous. A permanently lost job is a worse failure than a rare double-run of an idempotent step, so
+     * worker's job is re-delivered — which claim renewal ({@link #startClaimRenewal}) rules out on an
+     * adapter that supports it, and which is why {@link DEFAULT_STALLED_THRESHOLD_MS} is deliberately
+     * generous on one that doesn't. A permanently lost job is a worse failure than a rare double-run of an idempotent step, so
      * this defaults ON; set `stalledCheckIntervalMs: 0` to opt out.
      *
      * Feature-detected: an adapter that doesn't implement `recoverStalledJobs` (not every driver does)
@@ -415,13 +998,10 @@ export class QueueTransport {
                     // `@adonisjs/queue` 0.7 (`@boringnode/queue` >= 0.8) no longer fails a job that stalled too
                     // many times: it keeps it `active`, re-leases it to us and returns it in `exceeded`, so the
                     // caller fails it. Left alone it would stay `active` forever. 0.6 returns a count and has
-                    // already failed them — nothing to do.
+                    // already failed (deleted) them — nothing to do, and nothing left to bridge.
                     if (outcome && typeof outcome === 'object') {
-                        for (const job of outcome.exceeded) {
-                            await this.#adapter
-                                .failJob(jobHandle(job), queue, new Error(`stalled more than ${this.#maxStalledCount} times`))
-                                .catch((err) => this.#onError(err));
-                        }
+                        for (const job of outcome.exceeded)
+                            await this.#failPoisonJob(queue, job);
                     }
                 }
                 catch (err) {
@@ -432,6 +1012,100 @@ export class QueueTransport {
         });
         return pollers;
     }
+    /**
+     * Fail a job that stalled more than `maxStalledCount` times and, for a STEP task, publish a failed
+     * {@link StepResult} for it — the queue twin of the BullMQ transport's `worker.on('failed')` bridge.
+     * No handler produced a result for that step (its workers kept dying), so without one the run would
+     * wait on its `pending` checkpoint forever. The result is RETRYABLE: the engine re-dispatches while
+     * the step has attempts left, then fails it into the workflow's error path. Only after `failJob`
+     * succeeded (a lost lease means another sweep owns the job), and only for task queues: a stalled
+     * result/heartbeat/control job carries no step to fail, and a workflow-turn payload has no step
+     * identity, so both are just failed.
+     */
+    async #failPoisonJob(queue, job) {
+        const reason = `stalled more than ${this.#maxStalledCount} times`;
+        let failed;
+        try {
+            failed = await this.#adapter.failJob(jobHandle(job), queue, new Error(reason));
+        }
+        catch (err) {
+            this.#onError(err);
+            return;
+        }
+        if (failed === false || !this.#taskQueues.has(queue))
+            return;
+        let identity;
+        try {
+            identity = failedTaskIdentity(fromJson(job.payload));
+        }
+        catch {
+            return; // an unparseable payload: nothing safe to publish
+        }
+        if (!identity)
+            return;
+        try {
+            await this.#adapter.pushOn(this.#resultsQueue(), this.#job('result', infrastructureFailure(identity, reason)));
+        }
+        catch (err) {
+            this.#onError(err);
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // claim renewal
+    // ---------------------------------------------------------------------------
+    /**
+     * Renew every claim this instance is still working on, each half {@link #stalledThresholdMs} —
+     * the heartbeat `@boringnode/queue`'s own `Worker` runs, which this transport bypasses (see the
+     * class header) exactly as it bypasses that class's stalled recovery.
+     *
+     * WHY: without it, a claim's age is the step's elapsed runtime, so the threshold had to exceed the
+     * longest step — a 30min default that also left a dead worker's job unrecovered for 30min, while a
+     * step that legitimately ran longer was re-delivered and ran twice. Renewing makes the age measure
+     * the worker's silence instead, so a short threshold recovers crashes fast without ever re-running a
+     * healthy long step.
+     *
+     * Each claim is renewed with the handle it is finalized with (lease on 0.9, job id on 0.6). The timer is `unref`'d, and a round still in flight skips the next tick instead of stacking.
+     */
+    #startClaimRenewal() {
+        const intervalMs = Math.max(Math.floor(this.#stalledThresholdMs / 2), 1);
+        this.#renewTimer = setInterval(() => {
+            if (this.#renewal)
+                return;
+            const round = this.#renewOpenClaims().finally(() => {
+                if (this.#renewal === round)
+                    this.#renewal = undefined;
+            });
+            this.#renewal = round;
+        }, intervalMs);
+        this.#renewTimer.unref?.();
+    }
+    async #renewOpenClaims() {
+        if (this.#openJobs.size === 0)
+            return;
+        const byQueue = new Map();
+        for (const { queue, handle } of this.#openJobs.values()) {
+            const leases = byQueue.get(queue) ?? [];
+            leases.push(handle);
+            byQueue.set(queue, leases);
+        }
+        for (const [queue, leases] of byQueue) {
+            try {
+                const renewed = await this.#adapter.renewJobs?.(queue, leases);
+                // A job that finished normally during the round-trip was finalized, not lost: count only the
+                // claims still open under the same handle, or a healthy completion would raise a false alarm.
+                const stillHeld = leases.filter((lease) => this.#openJobs.get(typeof lease === 'string' ? lease : lease.id)?.handle === lease).length;
+                // Fewer renewed than still held: the sweep already re-delivered those claims (this worker went
+                // silent past the threshold), so another worker may be running the same step right now. The
+                // handler can't be aborted — durable steps are idempotent — but the double run must be loud.
+                if (typeof renewed === 'number' && renewed < stillHeld) {
+                    this.#onError(new Error(`[QueueTransport] lost ${stillHeld - renewed} of ${stillHeld} claim(s) on "${queue}" before renewing them — the jobs were re-delivered and may run twice; raise stalledThresholdMs if this worker blocks its event loop`));
+                }
+            }
+            catch (err) {
+                this.#onError(err);
+            }
+        }
+    }
     /**
      * Stop every poll loop, WAIT (bounded) for the job already in a handler's hands, and hand any
      * still-unfinished job back to the broker before destroying the adapter. Without the wait, a
@@ -443,13 +1117,49 @@ export class QueueTransport {
         this.#pollers.stopAll();
         this.#reclaimPollers?.stopAll();
         this.#taskLoops.clear();
-        await this.#pollers.drain(5_000);
+        this.#controller.stop();
+        if (this.#workerHeartbeatTimer)
+            clearInterval(this.#workerHeartbeatTimer);
+        // A clean stop withdraws this instance's liveness keys at once, so the dashboard and
+        // `listWorkerGroups` drop it now instead of after the 35s TTL. In-flight beats land first, or one
+        // could re-create a key after its removal.
+        if (this.#heartbeatTokens.size > 0) {
+            await Promise.allSettled([...this.#livenessWrites]);
+            const keys = [...this.#heartbeatTokens].map((t) => workerHeartbeatKey(this.#effectivePrefix(), t, this.#instanceId));
+            this.#heartbeatTokens.clear();
+            await Promise.resolve()
+                .then(() => this.#workerKeys.remove(keys))
+                .catch((err) => this.#onError(err));
+        }
+        this.#workerHeartbeatTimer = undefined;
+        const drainDeadline = Date.now() + this.#closeTimeoutMs;
+        await this.#pollers.drain(this.#closeTimeoutMs);
+        // Jobs handed off by a multi-slot loop are not part of a poll tick: wait for them too, within the
+        // same budget.
+        if (this.#runningJobs.size > 0) {
+            const left = Math.max(drainDeadline - Date.now(), 0);
+            let timer;
+            await Promise.race([
+                Promise.allSettled([...this.#runningJobs]),
+                new Promise((r) => {
+                    timer = setTimeout(r, left);
+                }),
+            ]);
+            clearTimeout(timer);
+        }
+        // Renewal keeps running through the drain (a job finishing inside it must keep its claim), then
+        // stops before the leftovers are handed back.
+        if (this.#renewTimer)
+            clearInterval(this.#renewTimer);
+        this.#renewTimer = undefined;
+        await this.#renewal;
         // Anything still open after the bounded drain (a long step): re-deliver it now so another
         // worker picks it up immediately instead of waiting out the stalled threshold.
         for (const [jobId, { queue, handle }] of [...this.#openJobs]) {
             this.#openJobs.delete(jobId);
             await this.#adapter.retryJob(handle, queue, new Date()).catch((err) => this.#onError(err));
         }
+        await this.#closeInboxes();
         await this.#adapter.destroy();
     }
 }