@adonis-agora/durable 0.49.1 → 0.50.0

package/dist/src/transports/db.js

@@ -1,8 +1,13 @@
 import { randomUUID } from 'node:crypto';
-import { Pollers } from '../pollers.js';
-import { runStepHandler } from '../protocol.js';
+import { atLeastOne, Pollers } from '../pollers.js';
+import { runStepHandler, withTurnHeartbeat } from '../protocol.js';
 import { sanitizeQueueToken, tenantGroup } from '../tenant-group.js';
-import { createDurableTransportTables, TRANSPORT_TABLES } from './db-schema.js';
+import { AdaptiveController } from './adaptive-concurrency.js';
+import { effectivePrefix, workerHeartbeatKey } from './bullmq/naming.js';
+import { heartbeatKeyValue, parseHeartbeatValue, WORKER_HEARTBEAT_INTERVAL_MS, WORKER_HEARTBEAT_TTL_SECONDS, } from './bullmq/serialization.js';
+import { DbInboxRegistry, DEFAULT_DB_INBOX_SWEEP_INTERVAL_MS, DEFAULT_DB_INBOX_TTL_MS, DEFAULT_DB_TENANT_SUBSCRIBERS_CACHE_MS, } from './db-remote.js';
+import { createDurableTransportTables, MESSAGE_KINDS, TRANSPORT_TABLES, } from './db-schema.js';
+import { fanOutDecision, infrastructureFailure } from './task-failure.js';
 import { cachedLiveKeys, descriptorsOn, liveGroupsOf, workerKeyWriter, } from './worker-keys.js';
 import { lucidWorkerKeys } from './worker-keys-db.js';
 /**
@@ -25,8 +30,48 @@ function fromJson(value) {
 const DEFAULT_POLL_INTERVAL_MS = 200;
 /** Default lease window (ms): a claimed-but-not-deleted row is reclaimable after this (crash recovery). */
 const DEFAULT_LEASE_MS = 30_000;
-/** Default max rows claimed per poll. */
+/** Default max rows claimed per poll (results / heartbeats / control; the task loop claims only as
+ *  many rows as it has free slots, never more than this). */
 const DEFAULT_BATCH_SIZE = 20;
+/** Idle backoff for the task, heartbeat and control loops (the results loop is exempt): after
+ *  consecutive empty polls the sleep doubles up to this many poll intervals. Same default as the
+ *  queue transport. */
+const DEFAULT_IDLE_BACKOFF_MAX = 4;
+/** How many times a task row's lapsed claim may be taken over before the row is treated as poison
+ *  (its workers keep dying on it) and failed instead of re-run. Same default as the queue transport. */
+const DEFAULT_MAX_STALLED_COUNT = 3;
+/** Key prefix of the worker liveness keys — the one a `WorkerRuntime` uses by default, so both kinds of
+ *  beat land in the same keyspace and `groupHealth` reads them alike. */
+const WORKER_KEY_PREFIX = 'durable';
+/** Default budget (ms) `close()` waits for in-flight work before handing the rest back. */
+const DEFAULT_CLOSE_TIMEOUT_MS = 5_000;
+/** The `priority` column is a 32-bit integer on every dialect. */
+const PRIORITY_MIN = -2_147_483_648;
+const PRIORITY_MAX = 2_147_483_647;
+/**
+ * The `priority` column value for the engine's dispatch `priority` (higher = more urgent): rounded and
+ * clamped to the column's range, and `0` — "normal" — when absent, which is where the queue
+ * transport's broker mapping puts an unprioritised job too.
+ */
+export function toDbPriority(priority) {
+    if (priority == null || Number.isNaN(priority))
+        return 0;
+    return Math.min(PRIORITY_MAX, Math.max(PRIORITY_MIN, Math.round(priority)));
+}
+const MISSING_COLUMN = {
+    reclaims: {
+        effect: 'poison tasks are not bounded (maxStalledCount is off)',
+        ddl: 'integer, not null, default 0',
+    },
+    priority: {
+        effect: 'dispatch priority is ignored and tasks are claimed oldest first',
+        ddl: 'integer, not null, default 0',
+    },
+    kind: {
+        effect: 'remote workflow turns cannot be dispatched on it',
+        ddl: "string, not null, default 'step'",
+    },
+};
 const DEFAULT_LOG = (err) => console.error('[DbTransport] poll failed', err);
 /**
  * A poll-based, DB-table-backed {@link Transport} (and best-effort {@link ControlPlane}) over
@@ -45,6 +90,12 @@ const DEFAULT_LOG = (err) => console.error('[DbTransport] poll failed', err);
  * {@link ControlPlane} here is single-consumer (one row → one poller), correct for a single engine
  * instance but NOT a true broadcast.
  *
+ * It also carries remote workflow turns (`dispatchWorkflowTask` / `handleWorkflow` / `onDecision`,
+ * streamed step events) and the store-less tenant-pod protocol (start-run, run-request, run replies,
+ * tenant events), so a `tenant` pod can reach a `control-plane` over the database alone. Those ride the
+ * `durable_transport_messages` table: run replies go to the requester only, tenant events to every
+ * live subscriber of the tenant, decisions to the engine that dispatched the turn — see `db-remote.ts`.
+ *
  * Run one instance engine-side (`onResult` + `dispatch`) and one per worker process (`handle()` for
  * its group). Call {@link start} to begin the engine-side pollers; `handle()` auto-starts the task
  * loop. The wire payloads are the documented `RemoteTask`/`StepResult` JSON.
@@ -80,8 +131,63 @@ export class DbTransport {
     #readWorkerKeys;
     #handlers = new Map();
     #pollers;
+    /** When set (see {@link deferConsumers}), {@link #startLoop} parks its loop until
+     *  {@link startConsumers}. Dispatching is never deferred; only consumption is. */
+    #consumersDeferred = false;
+    #deferredStarts = [];
     #schemaReady;
     #taskLoop;
+    /** Fixed or adaptive task concurrency, plus the in-flight/latency/RSS/CPU telemetry the liveness
+     *  key carries. One controller for the task loop of this instance, as on the queue transport. */
+    #controller;
+    /** A task loop parked on "every slot busy", woken when an adaptive move raises the limit. */
+    #limitWaiters = new Set();
+    /** Every task row still in a handler — the slots in use, and what `stop()` waits for (bounded). */
+    #runningTasks = new Set();
+    #resultConcurrency;
+    #idleBackoffMax;
+    #maxStalledCount;
+    #onError;
+    /** Claims this instance holds, per table: claim token → the row ids still held under it. Every one
+     *  is renewed on {@link #renewTimer}, so no held row ages into another instance's reclaim. */
+    #held = new Map();
+    #renewTimer;
+    /** The renewal round in flight, so a slow database never stacks rounds. */
+    #renewal;
+    /** Whether the tasks table has each adaptive column (resolved once per column). */
+    #taskColumns = new Map();
+    /** Claim tokens `close()` handed back — a handler still running under one learns why it lost it. */
+    #handedBack = new Set();
+    #closeTimeoutMs;
+    /** Routing tokens this instance beats a liveness key for (one per handled step name). */
+    #heartbeatTokens = new Set();
+    #workerHeartbeatIntervalMs;
+    #workerHeartbeatTimer;
+    /** Liveness-key writes still in flight — `close()` lets them land before withdrawing the keys. */
+    #livenessWrites = new Set();
+    /** Routing tokens whose liveness-key write is still in flight (see `#startWorkerHeartbeat`). */
+    #beatsInFlight = new Set();
+    /** Workflow turn consumers by workflow name — a `workflow` task row routes here. */
+    #workflowTurns = new Map();
+    /** Every `onDecision` handler (an engine subscribes one; it applies a decision by run id). */
+    #decisionHandlers = new Set();
+    /** Which message loops run (each starts once). */
+    #messageLoops = new Set();
+    #runReplyHandler;
+    /** Control-plane side: correlation id (requestId, or a start-run's runId) → the requester's instance
+     *  id, held while the request's handler runs so {@link publishRunReply} knows whom to address. */
+    #replyRoutes = new Map();
+    #tenantEventHandlers = new Map();
+    /** Serializes {@link publishTenantEvent} so each subscriber gets one publisher's events in order. */
+    #tenantPublishChain = Promise.resolve();
+    #inboxes;
+    #inboxTtlMs;
+    #inboxSweepIntervalMs;
+    #sweepTimer;
+    #sweeping;
+    /** Whether the messages table exists (resolved once), and whether its absence was reported. */
+    #messagesTable;
+    #messagesTableReported = false;
     constructor(options) {
         this.#db = options.db;
         this.#partition = options.partition;
@@ -94,7 +200,41 @@ export class DbTransport {
         this.#autoCreate = options.autoCreate ?? true;
         this.#instanceId = options.instanceId ?? randomUUID();
         this.#workerKeysCacheMs = options.workerKeysCacheMs ?? 5_000;
-        this.#pollers = new Pollers(this.#pollIntervalMs, DEFAULT_LOG);
+        this.#onError = options.onError ?? DEFAULT_LOG;
+        this.#closeTimeoutMs =
+            options.closeTimeoutMs !== undefined && options.closeTimeoutMs >= 0
+                ? options.closeTimeoutMs
+                : DEFAULT_CLOSE_TIMEOUT_MS;
+        this.#pollers = new Pollers(this.#pollIntervalMs, this.#onError);
+        this.#controller = new AdaptiveController({
+            ...(options.concurrency !== undefined ? { concurrency: options.concurrency } : {}),
+            apply: () => {
+                for (const wake of this.#limitWaiters)
+                    wake();
+            },
+        });
+        this.#resultConcurrency = atLeastOne(options.resultConcurrency, 1);
+        this.#idleBackoffMax = atLeastOne(options.idleBackoffMax, DEFAULT_IDLE_BACKOFF_MAX);
+        const maxStalled = options.maxStalledCount;
+        this.#maxStalledCount =
+            maxStalled !== undefined && !Number.isNaN(maxStalled) && maxStalled >= 0
+                ? Math.floor(maxStalled)
+                : DEFAULT_MAX_STALLED_COUNT;
+        this.#workerHeartbeatIntervalMs = atLeastOne(options.workerHeartbeatIntervalMs, WORKER_HEARTBEAT_INTERVAL_MS);
+        this.#inboxTtlMs = atLeastOne(options.inboxTtlMs, DEFAULT_DB_INBOX_TTL_MS);
+        this.#inboxSweepIntervalMs =
+            options.inboxSweepIntervalMs !== undefined && Number.isFinite(options.inboxSweepIntervalMs)
+                ? Math.max(0, options.inboxSweepIntervalMs)
+                : DEFAULT_DB_INBOX_SWEEP_INTERVAL_MS;
+        this.#inboxes = new DbInboxRegistry({
+            store: () => this.#workerKeys(),
+            prefix: () => this.#keyPrefix(),
+            instanceId: this.#instanceId,
+            ttlMs: this.#inboxTtlMs,
+            subscribersCacheMs: options.tenantSubscribersCacheMs ?? DEFAULT_DB_TENANT_SUBSCRIBERS_CACHE_MS,
+            onError: this.#onError,
+            now: () => this.#now(),
+        });
     }
     /** Stable id stamped on heartbeats, control `from`, and `claimed_by`. */
     get instanceId() {
@@ -133,6 +273,7 @@ export class DbTransport {
     // ───────────────────────────────────────────────────────────────────────────
     async dispatch(task) {
         await this.#ensureSchema();
+        const withPriority = await this.#hasTaskColumn('priority');
         // Idempotent: a redelivered dispatch for the same step_id is ignored (the PK already exists).
         // `onConflict().ignore()` is a Knex feature not surfaced on Lucid's insert contract, so we use
         // the underlying Knex builder (`.knexQuery`).
@@ -151,6 +292,7 @@ export class DbTransport {
             context: toJson(task.context),
             transport: task.transport ?? null,
             attempt: task.attempt,
+            ...(withPriority ? { priority: toDbPriority(task.priority) } : {}),
             claimed_by: null,
             claimed_at: null,
             created_at: this.#now(),
@@ -161,26 +303,100 @@ export class DbTransport {
     // ───────────────────────────────────────────────────────────────────────────
     // worker side — register a handler, claim + run tasks, write results
     // ───────────────────────────────────────────────────────────────────────────
+    /**
+     * Park every consumer loop registered from now on ({@link handle}, {@link onResult},
+     * {@link onHeartbeat}, {@link onControl}) until {@link startConsumers}. Task, result, heartbeat and
+     * control rows are claimed by ONE consumer each, so a process not built to be a worker — a
+     * `node ace` command, a REPL — would otherwise claim rows it then dies holding (re-delivered only
+     * after the lease expires) and steal results addressed to the long-lived engine. The same gate as
+     * the `queue` and `bullmq` transports; the provider calls it per the `consumers` config.
+     */
+    deferConsumers() {
+        this.#consumersDeferred = true;
+    }
+    /** Start every loop parked by {@link deferConsumers} and stop deferring (`durable:work` calls it via
+     *  `engine.startConsumers()`). Idempotent; a no-op when consumption was never deferred. */
+    startConsumers() {
+        if (!this.#consumersDeferred)
+            return;
+        this.#consumersDeferred = false;
+        for (const start of this.#deferredStarts.splice(0))
+            start();
+    }
+    /** `#pollers.start`, unless consumption is deferred: then park it, returning a handle that still
+     *  stops the loop whether it is started later or never. */
+    #startLoop(tick, idleBackoffMax) {
+        if (!this.#consumersDeferred)
+            return this.#pollers.start(tick, { idleBackoffMax });
+        let cancelled = false;
+        let live;
+        this.#deferredStarts.push(() => {
+            if (!cancelled)
+                live = this.#pollers.start(tick, { idleBackoffMax });
+        });
+        return {
+            stop: () => {
+                cancelled = true;
+                live?.stop();
+            },
+        };
+    }
+    /** Run `start` once this instance really consumes: now, or at {@link startConsumers} when deferred. */
+    #whenConsuming(start) {
+        if (this.#consumersDeferred)
+            this.#deferredStarts.push(start);
+        else
+            start();
+    }
     /**
      * Register a step handler (worker side). Records this name's routing token
      * (`tenantGroup(sanitizeQueueToken(name), partition)`) as served and starts the shared task poll
      * loop on the first call — it claims task rows for EVERY served token (matching the token the
-     * engine dispatches by), runs each through {@link runStepHandler}, and writes the result.
+     * engine dispatches by), runs each through {@link runStepHandler}, and writes the result. The loop
+     * keeps up to {@link DbTransportOptions.concurrency} rows in flight, and each served token gets a
+     * liveness key once the loop consumes (a deferred, producer-only process advertises nothing).
      */
     handle(name, fn) {
         this.#handlers.set(name, fn);
-        this.#servedTokens.add(tenantGroup(sanitizeQueueToken(name), this.#partition));
+        this.#serve(name);
+    }
+    /**
+     * Register a WORKFLOW turn consumer (worker side). Shares the task loop — and the routing token —
+     * of a step of the same name: {@link dispatchWorkflowTask} writes a task row of kind `workflow`, which
+     * runs `turn(task)` in a slot like any step, and its {@link WorkflowDecision} is written to the
+     * messages table, addressed to the engine instance that dispatched the turn. Re-registering swaps
+     * the turn fn.
+     */
+    handleWorkflow(name, turn) {
+        this.#workflowTurns.set(name, turn);
+        this.#serve(name);
+    }
+    /** Serve `name`'s routing token on the shared task loop (started on the first call). */
+    #serve(name) {
+        const token = tenantGroup(sanitizeQueueToken(name), this.#partition);
+        const isNew = !this.#servedTokens.has(token);
+        this.#servedTokens.add(token);
         if (!this.#taskLoop) {
-            this.#taskLoop = this.#pollers.start(async () => (await this.#drainTasks()) > 0);
+            this.#taskLoop = this.#startLoop(() => this.#taskTick(), this.#idleBackoffMax);
+            this.#whenConsuming(() => {
+                for (const served of this.#servedTokens)
+                    this.#beginServing(served);
+            });
+        }
+        else if (isNew) {
+            this.#whenConsuming(() => this.#beginServing(token));
         }
     }
     /** Worker side: publish a liveness heartbeat for an in-flight long step. */
     async heartbeat(beat) {
         await this.#ensureSchema();
-        await this.#client().table(TRANSPORT_TABLES.heartbeats).insert({
+        await this.#client()
+            .table(TRANSPORT_TABLES.heartbeats)
+            .insert({
             run_id: beat.runId,
             seq: beat.seq,
-            step_id: beat.stepId,
+            // '' for a run-scoped (workflow-turn) beat: the column is NOT NULL.
+            step_id: beat.stepId ?? '',
             grp: beat.group,
             namespace: this.#namespaceValue(),
             claimed_by: null,
@@ -188,56 +404,279 @@ export class DbTransport {
             created_at: this.#now(),
         });
     }
-    async #drainTasks() {
+    /**
+     * One task-loop round: claim as many rows as there are free slots (never more — a row claimed but
+     * not running is a row an idle sibling can't run, and one whose lease nobody is using), hand each
+     * to {@link #runTask}, and report work so the loop re-ticks at once. With every slot busy it waits
+     * for one to free (or the adaptive limit to move) instead of polling.
+     */
+    async #taskTick() {
         if (this.#servedTokens.size === 0)
-            return 0;
-        const { rows, token } = await this.#claim(TRANSPORT_TABLES.tasks, [
-            ...this.#servedTokens,
+            return false;
+        const free = this.#controller.limit - this.#runningTasks.size;
+        if (free <= 0) {
+            await this.#slotFreed();
+            return true;
+        }
+        const [countReclaims, byPriority] = await Promise.all([
+            this.#hasTaskColumn('reclaims'),
+            this.#hasTaskColumn('priority'),
         ]);
+        const { rows, token } = await this.#claim(TRANSPORT_TABLES.tasks, Math.min(free, this.#batchSize), [...this.#servedTokens], 'step_id', { countReclaims, byPriority });
+        if (rows.length === 0)
+            return false;
+        if (this.#pollers.closed) {
+            // close() began while this round was claiming: don't start work it would only abandon.
+            await this.#handBack(TRANSPORT_TABLES.tasks, [token]);
+            return false;
+        }
+        // `free` was read before the claim's round trips. The adaptive limit may have dropped meanwhile
+        // (the RAM brake fires on its own timer), and a round sized for the old limit would run past the
+        // new one. Un-claim what no longer fits, so the slot loop never runs more than the LIVE limit.
+        // (A lapsed row handed back this way keeps the reclaim its takeover counted: rare — a limit drop
+        // inside one claim round — and only ever early toward the poison bound, never past it unnoticed.)
+        const fits = Math.max(0, this.#controller.limit - this.#runningTasks.size);
+        if (rows.length > fits) {
+            const excess = rows.splice(fits);
+            await this.#client()
+                .knexQuery()
+                .from(TRANSPORT_TABLES.tasks)
+                .where('claimed_by', token)
+                .whereIn('step_id', excess.map((row) => row.step_id))
+                .update({ claimed_by: null, claimed_at: null });
+            if (rows.length === 0)
+                return true;
+        }
+        this.#hold(TRANSPORT_TABLES.tasks, token, rows.map((row) => row.step_id));
         for (const row of rows) {
-            const task = {
-                runId: row.run_id,
-                seq: Number(row.seq),
-                stepId: row.step_id,
-                name: row.name,
-                group: row.grp,
-                input: fromJson(row.input),
-                attempt: Number(row.attempt),
-                ...(row.traceparent != null ? { traceparent: row.traceparent } : {}),
-                ...(row.context != null ? { context: fromJson(row.context) } : {}),
-                ...(row.transport != null ? { transport: row.transport } : {}),
-            };
-            // Renew this row's claim NOW (rows after the first have aged while earlier ones ran) and
-            // KEEP renewing while the handler runs — the claim used to be stamped once for the whole
-            // batch and never refreshed, so any step longer than `leaseMs` (30s default) was reclaimed
-            // by a sibling worker mid-flight and GUARANTEED to double-run. A merely-slow worker now
-            // holds its claim for the step's whole duration; a crashed one still expires and is
-            // reclaimed.
-            const renew = () => {
-                void this.#client()
-                    .knexQuery()
-                    .from(TRANSPORT_TABLES.tasks)
-                    .where('step_id', task.stepId)
-                    .where('claimed_by', token)
-                    .update({ claimed_at: this.#now() })
-                    .catch(() => undefined);
-            };
-            renew();
-            const renewTimer = setInterval(renew, Math.max(1_000, Math.floor(this.#leaseMs / 2)));
-            renewTimer.unref?.();
+            const run = this.#runTask(row, token);
+            this.#runningTasks.add(run);
+            void run.finally(() => this.#runningTasks.delete(run));
+        }
+        return true;
+    }
+    /** Resolve when a running task settles or the adaptive controller moves the limit. */
+    async #slotFreed() {
+        let wake = () => { };
+        const moved = new Promise((resolve) => {
+            wake = resolve;
+        });
+        this.#limitWaiters.add(wake);
+        try {
+            await Promise.race([...this.#runningTasks, moved]);
+        }
+        finally {
+            this.#limitWaiters.delete(wake);
+        }
+    }
+    /**
+     * Run one claimed task row and settle it. A row whose claim lapsed more than
+     * {@link DbTransportOptions.maxStalledCount} times is bridged to a failed result instead of run.
+     * Otherwise the claim is re-checked first (a row is run only while this instance holds it), and the
+     * result is published only by the instance still holding the claim (see {@link #settleTask}).
+     * The claim is renewed for the whole run with every other held claim (see {@link #renewHeld}).
+     */
+    async #runTask(row, token) {
+        if (row.kind === 'workflow')
+            return this.#runWorkflowRow(row, token);
+        const task = {
+            runId: row.run_id,
+            seq: Number(row.seq),
+            stepId: row.step_id,
+            name: row.name,
+            group: row.grp,
+            input: fromJson(row.input),
+            attempt: Number(row.attempt),
+            ...(row.traceparent != null ? { traceparent: row.traceparent } : {}),
+            ...(row.context != null ? { context: fromJson(row.context) } : {}),
+            ...(row.transport != null ? { transport: row.transport } : {}),
+        };
+        try {
+            if (row.reclaims != null && Number(row.reclaims) > this.#maxStalledCount) {
+                // Its workers keep dying on it (or blocking past the lease): running it again would just
+                // stall the next one. Fail it RETRYABLE so the engine's durable retry decides.
+                this.#release(TRANSPORT_TABLES.tasks, token, task.stepId);
+                await this.#settleTask(task.stepId, token, infrastructureFailure(task, `claim lapsed more than ${this.#maxStalledCount} times`));
+                return;
+            }
+            if (!(await this.#stillHolds(task.stepId, token))) {
+                this.#release(TRANSPORT_TABLES.tasks, token, task.stepId);
+                this.#onError(new Error(`[DbTransport] lost the claim on task ${task.stepId} before running it — another worker took it over; not running it here`));
+                return;
+            }
+            const startedAt = Date.now();
+            this.#controller.onStart();
+            let ok = false;
             try {
                 const result = await runStepHandler(task, this.#handlers.get(task.name), (beat) => this.heartbeat(beat));
-                await this.#insertResult(result);
+                ok = result.status === 'completed';
+                // Stop renewing before settling: the settle deletes the row, and a renewal racing it would
+                // count the row as lost.
+                this.#release(TRANSPORT_TABLES.tasks, token, task.stepId);
+                if (!(await this.#settleTask(task.stepId, token, result))) {
+                    if (this.#handedBack.has(token)) {
+                        this.#onError(new Error(`[DbTransport] task ${task.stepId} outlived close()'s ${this.#closeTimeoutMs}ms drain and was handed back, so another worker may have run it too; this run's result was dropped`));
+                        return;
+                    }
+                    this.#onError(new Error(`[DbTransport] lost the claim on task ${task.stepId} while running it (this worker went silent past leaseMs) — another worker took it over and may run it too; this run's result was dropped. Raise leaseMs if steps block the event loop`));
+                }
             }
             finally {
-                clearInterval(renewTimer);
+                this.#controller.onSettle(Date.now() - startedAt, ok, 'step');
             }
-            await this.#client().from(TRANSPORT_TABLES.tasks).where('step_id', task.stepId).delete();
         }
-        return rows.length;
+        catch (err) {
+            // A database error settling the row: it stays claimed, unrenewed, and is reclaimed after the
+            // lease — at-least-once, like a crash.
+            this.#onError(err);
+        }
+        finally {
+            this.#release(TRANSPORT_TABLES.tasks, token, task.stepId);
+        }
+    }
+    /**
+     * Run one claimed `workflow` task row: replay the turn and settle it — delete the row and write its
+     * {@link WorkflowDecision}, addressed to the engine that dispatched it, in ONE transaction and only
+     * while this round still holds the claim (as a step's result). An unknown workflow or a throwing turn
+     * still yields a `failed` decision so the run settles. A row past the poison bound is deleted without
+     * a decision, as on the other transports: a turn carries no step to fail, and the turn's liveness
+     * window (`remoteAdvanceSilenceMs` / the executor's `timeoutMs`) re-drives it.
+     */
+    async #runWorkflowRow(row, token) {
+        const id = row.step_id;
+        try {
+            let task;
+            let replyTo;
+            try {
+                const payload = fromJson(row.input);
+                if (!payload)
+                    throw new Error('empty payload');
+                ({ replyTo, ...task } = payload);
+            }
+            catch (err) {
+                this.#release(TRANSPORT_TABLES.tasks, token, id);
+                await this.#deleteClaimed(TRANSPORT_TABLES.tasks, 'step_id', id, token);
+                this.#onError(new Error(`[DbTransport] dropped workflow task ${id}: unreadable payload (${String(err)})`));
+                return;
+            }
+            if (row.reclaims != null && Number(row.reclaims) > this.#maxStalledCount) {
+                this.#release(TRANSPORT_TABLES.tasks, token, id);
+                await this.#deleteClaimed(TRANSPORT_TABLES.tasks, 'step_id', id, token);
+                this.#onError(new Error(`[DbTransport] dropped workflow task ${id} (run ${task.runId}): its claim lapsed more than ${this.#maxStalledCount} times. No decision is written: as on bullmq and queue, the turn is re-driven by the executor's timeoutMs or by recovery`));
+                return;
+            }
+            if (!(await this.#stillHolds(id, token))) {
+                this.#release(TRANSPORT_TABLES.tasks, token, id);
+                this.#onError(new Error(`[DbTransport] lost the claim on workflow task ${id} before running it — another worker took it over; not running it here`));
+                return;
+            }
+            const startedAt = Date.now();
+            this.#controller.onStart();
+            try {
+                const decision = await this.#replayTurn(task);
+                this.#release(TRANSPORT_TABLES.tasks, token, id);
+                if (!(await this.#settleWorkflow(id, token, decision, replyTo))) {
+                    this.#onError(new Error(this.#handedBack.has(token)
+                        ? `[DbTransport] workflow task ${id} outlived close()'s ${this.#closeTimeoutMs}ms drain and was handed back; this turn's decision was dropped`
+                        : `[DbTransport] lost the claim on workflow task ${id} while running it — another worker took it over; this turn's decision was dropped`));
+                }
+            }
+            finally {
+                // Only step durations feed the latency gradient; a turn still counts as in-flight work.
+                this.#controller.onSettle(Date.now() - startedAt, true, 'workflow');
+            }
+        }
+        catch (err) {
+            this.#onError(err);
+        }
+        finally {
+            this.#release(TRANSPORT_TABLES.tasks, token, id);
+        }
     }
-    async #insertResult(result) {
+    /** Replay one turn. Never throws: an unknown workflow or a throwing turn is a `failed` decision. */
+    async #replayTurn(task) {
+        const turn = this.#workflowTurns.get(task.workflow);
+        if (!turn) {
+            return {
+                taskId: task.taskId,
+                runId: task.runId,
+                status: 'failed',
+                commands: [],
+                error: { message: `no workflow registered for '${task.workflow}'`, code: 'no_workflow' },
+            };
+        }
+        try {
+            return await withTurnHeartbeat(task, (beat) => this.heartbeat(beat), () => turn(task));
+        }
+        catch (err) {
+            this.#onError(err);
+            return {
+                taskId: task.taskId,
+                runId: task.runId,
+                status: 'failed',
+                commands: [],
+                error: { message: err instanceof Error ? err.message : String(err) },
+            };
+        }
+    }
+    /** Delete the workflow row and write its decision in ONE transaction, only while `token` holds it. */
+    async #settleWorkflow(id, token, decision, replyTo) {
+        const settled = await this.#client().transaction(async (trx) => {
+            const deleted = await trx
+                .knexQuery()
+                .from(TRANSPORT_TABLES.tasks)
+                .where('step_id', id)
+                .where('claimed_by', token)
+                .delete();
+            if (Number(deleted) === 0)
+                return false;
+            await this.#insertMessages(MESSAGE_KINDS.decision, [{ recipient: replyTo ?? '', payload: decision }], trx);
+            return true;
+        });
+        if (settled && replyTo)
+            this.#noteAddressed();
+        return settled;
+    }
+    /** Delete one row of `table` while `token` still holds its claim. */
+    async #deleteClaimed(table, idCol, id, token) {
         await this.#client()
+            .knexQuery()
+            .from(table)
+            .where(idCol, id)
+            .where('claimed_by', token)
+            .delete();
+    }
+    /** Re-stamp the row's claim if this round still holds it; `false` once another instance took it. */
+    async #stillHolds(stepId, token) {
+        const updated = await this.#client()
+            .knexQuery()
+            .from(TRANSPORT_TABLES.tasks)
+            .where('step_id', stepId)
+            .where('claimed_by', token)
+            .update({ claimed_at: this.#now() });
+        return Number(updated) > 0;
+    }
+    /**
+     * Delete the task row and publish its result in ONE transaction, and only while `token` still holds
+     * the claim: an instance whose claim was taken over neither deletes the new holder's row nor writes
+     * a second result. Resolves whether this instance settled it.
+     */
+    async #settleTask(stepId, token, result) {
+        return this.#client().transaction(async (trx) => {
+            const deleted = await trx
+                .knexQuery()
+                .from(TRANSPORT_TABLES.tasks)
+                .where('step_id', stepId)
+                .where('claimed_by', token)
+                .delete();
+            if (Number(deleted) === 0)
+                return false;
+            await this.#insertResult(result, trx);
+            return true;
+        });
+    }
+    async #insertResult(result, client = this.#client()) {
+        await client
             .insertQuery()
             .table(TRANSPORT_TABLES.results)
             .knexQuery.insert({
@@ -260,57 +699,63 @@ export class DbTransport {
     // ───────────────────────────────────────────────────────────────────────────
     // worker → engine — the engine polls results + heartbeats
     // ───────────────────────────────────────────────────────────────────────────
+    /** Engine side: settle step results, up to {@link DbTransportOptions.resultConcurrency} at once. The
+     *  results loop never backs off when idle — a result is on the critical path of every step. */
     onResult(handler) {
-        this.#pollers.start(async () => (await this.#drainResults(handler)) > 0);
+        this.#startLoop(async () => (await this.#drain(TRANSPORT_TABLES.results, 'step_id', this.#resultConcurrency, async (row) => {
+            await handler({
+                runId: row.run_id,
+                seq: Number(row.seq),
+                stepId: row.step_id,
+                status: row.status,
+                output: fromJson(row.output),
+                error: fromJson(row.error),
+                startedAt: row.started_at == null ? undefined : Number(row.started_at),
+                events: fromJson(row.events),
+            });
+        })) > 0, 1);
     }
     onHeartbeat(handler) {
-        this.#pollers.start(async () => (await this.#drainHeartbeats(handler)) > 0);
-    }
-    async #drainResults(handler) {
-        const { rows } = await this.#claim(TRANSPORT_TABLES.results);
-        // Delete each row only AFTER its handler ran (at-least-once), but batch the deletes of a round
-        // into one `WHERE IN` — one round trip instead of one per row.
-        const done = [];
-        try {
-            for (const row of rows) {
-                const result = {
-                    runId: row.run_id,
-                    seq: Number(row.seq),
-                    stepId: row.step_id,
-                    status: row.status,
-                    output: fromJson(row.output),
-                    error: fromJson(row.error),
-                    startedAt: row.started_at == null ? undefined : Number(row.started_at),
-                    events: fromJson(row.events),
-                };
-                await handler(result);
-                done.push(result.stepId);
-            }
-        }
-        finally {
-            if (done.length) {
-                await this.#client().from(TRANSPORT_TABLES.results).whereIn('step_id', done).delete();
-            }
-        }
-        return rows.length;
+        this.#startLoop(async () => (await this.#drain(TRANSPORT_TABLES.heartbeats, 'id', 1, async (row) => {
+            await handler({
+                runId: row.run_id,
+                seq: Number(row.seq),
+                ...(row.step_id ? { stepId: row.step_id } : {}),
+                group: row.grp,
+            });
+        })) > 0, this.#idleBackoffMax);
     }
-    async #drainHeartbeats(handler) {
-        const { rows } = await this.#claim(TRANSPORT_TABLES.heartbeats, undefined, 'id');
+    /**
+     * Claim a batch of `table` rows, hand each to `each` (up to `concurrency` at once), then delete the
+     * handled rows in one `WHERE IN` — only AFTER their handler ran (at-least-once), and only while this
+     * round still holds their claim. The batch's claims are renewed until it is done, so a slow handler
+     * never lets the rows behind it age into another instance's reclaim. A throwing `each` stops the
+     * batch: the rows not handled stay claimed, unrenewed, and are re-delivered after the lease.
+     */
+    async #drain(table, idCol, concurrency, each, scope) {
+        const { rows, token } = await this.#claim(table, this.#batchSize, undefined, idCol, {
+            scope,
+        });
+        if (rows.length === 0)
+            return 0;
+        const idOf = (row) => row[idCol];
+        this.#hold(table, token, rows.map(idOf));
         const done = [];
         try {
-            for (const row of rows) {
-                await handler({
-                    runId: row.run_id,
-                    seq: Number(row.seq),
-                    stepId: row.step_id,
-                    group: row.grp,
-                });
-                done.push(row.id);
-            }
+            await eachLimited(rows, concurrency, async (row) => {
+                await each(row);
+                done.push(idOf(row));
+            });
         }
         finally {
+            this.#releaseToken(table, token);
             if (done.length) {
-                await this.#client().from(TRANSPORT_TABLES.heartbeats).whereIn('id', done).delete();
+                await this.#client()
+                    .knexQuery()
+                    .from(table)
+                    .whereIn(idCol, done)
+                    .where('claimed_by', token)
+                    .delete();
             }
         }
         return rows.length;
@@ -332,45 +777,434 @@ export class DbTransport {
         });
     }
     onControl(handler) {
-        this.#pollers.start(async () => (await this.#drainControl(handler)) > 0);
+        this.#startLoop(async () => (await this.#drain(TRANSPORT_TABLES.control, 'id', 1, async (row) => {
+            const msg = fromJson(row.payload);
+            if (msg)
+                handler(msg);
+        })) > 0, this.#idleBackoffMax);
     }
-    async #drainControl(handler) {
-        const { rows } = await this.#claim(TRANSPORT_TABLES.control, undefined, 'id');
-        const done = [];
+    // ───────────────────────────────────────────────────────────────────────────
+    // remote workflows + the store-less (tenant pod) protocol — the messages table
+    //
+    // Workflow turns ride the task table (kind `workflow`, next to the steps of the same token). Every
+    // other message is a row of `durable_transport_messages`, claimed and deleted like the other
+    // channels and told apart by `kind`. A row is either for ANY consumer of its kind (`recipient = ''`:
+    // step events, start-run, run-request) or addressed to ONE instance (`recipient = <instanceId>`: the
+    // reply to the requester, a tenant event to each live subscriber, a decision to the engine that
+    // dispatched the turn). See `db-remote.ts` for the subscriber registry and the janitor.
+    // ───────────────────────────────────────────────────────────────────────────
+    /**
+     * Whether the messages table exists — `autoCreate` creates it; a deployment that runs its own
+     * migrations needs `create_durable_transport_messages`. Resolved once. With `report`, its absence is
+     * said ONCE through `onError` (the consumer loops ask silently, so a deployment that never uses the
+     * protocol hears nothing).
+     */
+    async #messagesReady(report) {
+        let known = this.#messagesTable;
+        if (!known) {
+            known = this.#ensureSchema()
+                .then(() => this.#client().schema.hasTable(TRANSPORT_TABLES.messages))
+                .catch((err) => {
+                this.#messagesTable = undefined; // unknown yet (a transient error): ask again next time
+                this.#onError(err);
+                return false;
+            });
+            this.#messagesTable = known;
+        }
+        const has = await known;
+        if (!has && report && !this.#messagesTableReported) {
+            this.#messagesTableReported = true;
+            this.#onError(new Error(this.#messagesMissing()));
+        }
+        return has;
+    }
+    #messagesMissing() {
+        return `[DbTransport] the ${TRANSPORT_TABLES.messages} table does not exist, so remote workflows and store-less tenant pods do not work on this transport. Run the create_durable_transport_messages migration (node ace configure @adonis-agora/durable publishes it), or enable autoCreate`;
+    }
+    /** Insert `rows` of `kind` in one statement. */
+    async #insertMessages(kind, rows, client = this.#client()) {
+        if (rows.length === 0)
+            return;
+        const at = this.#now();
+        await client
+            .insertQuery()
+            .table(TRANSPORT_TABLES.messages)
+            .knexQuery.insert(rows.map((row) => ({
+            kind,
+            namespace: this.#namespaceValue(),
+            recipient: row.recipient ?? '',
+            reply_to: row.replyTo ?? null,
+            payload: JSON.stringify(row.payload),
+            claimed_by: null,
+            claimed_at: null,
+            created_at: at,
+        })));
+    }
+    /**
+     * Start the consumer loop of `kind` (once). `recipients` is re-read every poll. A loop over rows
+     * anyone may take is `shared`: parked while consumers are deferred, like every other consumer. One
+     * over rows addressed to this instance is not — nobody else could take them. A payload that does not
+     * parse is reported and dropped (never re-delivered forever).
+     */
+    #startMessageLoop(kind, { shared, recipients, idleBackoffMax, each, }) {
+        if (this.#messageLoops.has(kind))
+            return;
+        this.#messageLoops.add(kind);
+        const tick = async () => {
+            if (!(await this.#messagesReady(false)))
+                return false;
+            const handled = await this.#drain(TRANSPORT_TABLES.messages, 'id', 1, async (row) => {
+                let payload;
+                try {
+                    payload = JSON.parse(row.payload);
+                }
+                catch (err) {
+                    this.#onError(new Error(`[DbTransport] dropped ${kind} message ${row.id}: ${String(err)}`));
+                    return;
+                }
+                await each(payload, row);
+            }, { kind, recipients: recipients() });
+            return handled > 0;
+        };
+        if (shared)
+            this.#startLoop(tick, idleBackoffMax);
+        else
+            this.#pollers.start(tick, { idleBackoffMax });
+    }
+    /** This instance consumes rows addressed to it: keep its inbox key live (and delete what is left
+     *  addressed to it on close). */
+    #consumeInbox() {
+        void this.#inboxes.activateInbox();
+    }
+    /** This instance addressed a row to another one: run the janitor (once). */
+    #noteAddressed() {
+        if (this.#inboxSweepIntervalMs <= 0 || this.#sweepTimer)
+            return;
+        this.#sweepTimer = setInterval(() => {
+            if (this.#sweeping)
+                return;
+            const round = this.sweepInboxes()
+                .catch((err) => this.#onError(err))
+                .finally(() => {
+                if (this.#sweeping === round)
+                    this.#sweeping = undefined;
+            });
+            this.#sweeping = round;
+        }, this.#inboxSweepIntervalMs);
+        this.#sweepTimer.unref?.();
+    }
+    /**
+     * One janitor round: delete the addressed rows older than the inbox TTL whose recipient has no live
+     * inbox key — a requester or subscriber that died, an engine that crashed awaiting a decision. Rows
+     * younger than the TTL are never touched, so an instance that is just starting (or whose key write is
+     * a refresh late) keeps its rows. Reads everything from the database, so it needs no memory of what
+     * was addressed — any instance that addressed a row runs it, and a restart forgets nothing.
+     */
+    async sweepInboxes() {
+        if (!(await this.#messagesReady(false)))
+            return;
+        const cutoff = this.#now() - this.#inboxTtlMs;
+        const namespace = this.#namespaceValue();
+        const recipients = (await this.#client()
+            .knexQuery()
+            .from(TRANSPORT_TABLES.messages)
+            .distinct('recipient')
+            .where('namespace', namespace)
+            .whereNot('recipient', '')
+            .where('created_at', '<', cutoff)).map((r) => r.recipient);
+        if (recipients.length === 0)
+            return;
+        const alive = await this.#inboxes.liveInboxOwners();
+        const dead = recipients.filter((r) => !alive.has(r));
+        if (dead.length === 0)
+            return;
+        await this.#client()
+            .knexQuery()
+            .from(TRANSPORT_TABLES.messages)
+            .where('namespace', namespace)
+            .whereIn('recipient', dead)
+            .where('created_at', '<', cutoff)
+            .delete();
+    }
+    /**
+     * engine → workflow worker: a {@link WorkflowTask} as a task row of kind `workflow` on the workflow's
+     * routing token, where a {@link handleWorkflow} consumer claims it (with the token's steps, by
+     * priority). Stamped with this instance as the decision's recipient when it consumes decisions, so
+     * the decision comes back to the engine that dispatched the turn (any replica could apply it — the
+     * engine applies decisions durably by run id — but the dispatcher is the one known to be live).
+     * Idempotent per `taskId`. Throws when the schema lacks the tasks `kind` column or the messages table.
+     */
+    async dispatchWorkflowTask(task) {
+        await this.#ensureSchema();
+        const [hasKind, hasMessages, withPriority] = await Promise.all([
+            this.#hasTaskColumn('kind'),
+            this.#messagesReady(true),
+            this.#hasTaskColumn('priority'),
+        ]);
+        if (!hasMessages)
+            throw new Error(this.#messagesMissing());
+        if (!hasKind) {
+            throw new Error(`[DbTransport] ${TRANSPORT_TABLES.tasks} has no "kind" column, so remote workflow turns cannot be dispatched on it. Run the create_durable_transport_messages migration, or enable autoCreate`);
+        }
+        const payload = {
+            ...task,
+            ...(this.#decisionHandlers.size > 0 ? { replyTo: this.#instanceId } : {}),
+        };
+        await this.#client()
+            .insertQuery()
+            .table(TRANSPORT_TABLES.tasks)
+            .knexQuery.insert({
+            step_id: task.taskId,
+            run_id: task.runId,
+            seq: 0,
+            name: task.workflow,
+            grp: task.group,
+            namespace: this.#namespaceValue(),
+            input: toJson(payload),
+            traceparent: task.traceparent ?? null,
+            context: null,
+            transport: task.transport ?? null,
+            attempt: task.attempt,
+            kind: 'workflow',
+            ...(withPriority ? { priority: toDbPriority(task.priority) } : {}),
+            claimed_by: null,
+            claimed_at: null,
+            created_at: this.#now(),
+        })
+            .onConflict('step_id')
+            .ignore();
+    }
+    /**
+     * engine ← workflow worker: consume {@link WorkflowDecision}s. ONE loop per transport; every handler
+     * sees every decision (the engine applies one by run id, dropping any its run no longer awaits). It claims the
+     * decisions addressed to this instance — consumed even while consumers are deferred, as nobody else
+     * may take them — and, once consuming, the unaddressed ones (a worker that did not know whom to
+     * reply to). Never backs off: a decision is on the critical path of every remote workflow turn.
+     */
+    onDecision(handler) {
+        this.#decisionHandlers.add(handler);
+        if (this.#messageLoops.has(MESSAGE_KINDS.decision))
+            return;
+        this.#consumeInbox();
+        this.#startMessageLoop(MESSAGE_KINDS.decision, {
+            shared: false,
+            recipients: () => (this.#consumersDeferred ? [this.#instanceId] : [this.#instanceId, '']),
+            idleBackoffMax: 1,
+            each: async (decision) => {
+                // Every handler, concurrently; all settle before a failure redelivers the decision.
+                await fanOutDecision(this.#decisionHandlers, decision, this.#onError);
+            },
+        });
+    }
+    /** workflow worker → engine: a local step's lifecycle, for ONE engine instance to checkpoint. A
+     *  deployment without the messages table drops it (reported once): it is observability only. */
+    async dispatchStepEvent(event) {
+        if (!(await this.#messagesReady(true)))
+            return;
+        await this.#insertMessages(MESSAGE_KINDS.stepEvent, [{ payload: event }]);
+    }
+    /** engine ← workflow worker: consume streamed step events. Idempotent — a second call is a no-op. */
+    onStepEvent(handler) {
+        this.#startMessageLoop(MESSAGE_KINDS.stepEvent, {
+            shared: true,
+            recipients: () => [''],
+            idleBackoffMax: this.#idleBackoffMax,
+            each: (event) => handler(event),
+        });
+    }
+    /** tenant → control plane: a {@link StartRunMessage} for any control plane, with this instance as
+     *  the reply's recipient. Throws when the messages table is missing. */
+    async dispatchStartRun(msg) {
+        this.#ensureRunRepliesLoop();
+        if (!(await this.#messagesReady(true)))
+            throw new Error(this.#messagesMissing());
+        await this.#insertMessages(MESSAGE_KINDS.startRun, [
+            { replyTo: this.#instanceId, payload: msg },
+        ]);
+    }
+    /** control plane ← tenant: consume {@link StartRunMessage}s. Idempotent — a second call is a no-op. */
+    onStartRun(handler) {
+        this.#startMessageLoop(MESSAGE_KINDS.startRun, {
+            shared: true,
+            recipients: () => [''],
+            idleBackoffMax: this.#idleBackoffMax,
+            each: (msg, row) => this.#withReplyRoute(msg.runId, row.reply_to, () => handler(msg)),
+        });
+    }
+    /** tenant → control plane: a {@link RunRequest} for any control plane, with this instance as the
+     *  reply's recipient. Throws when the messages table is missing. */
+    async dispatchRunRequest(msg) {
+        this.#ensureRunRepliesLoop();
+        if (!(await this.#messagesReady(true)))
+            throw new Error(this.#messagesMissing());
+        await this.#insertMessages(MESSAGE_KINDS.runRequest, [
+            { replyTo: this.#instanceId, payload: msg },
+        ]);
+    }
+    /** control plane ← tenant: consume {@link RunRequest}s. Idempotent — a second call is a no-op. */
+    onRunRequest(handler) {
+        this.#startMessageLoop(MESSAGE_KINDS.runRequest, {
+            shared: true,
+            recipients: () => [''],
+            idleBackoffMax: this.#idleBackoffMax,
+            each: (msg, row) => this.#withReplyRoute(msg.requestId, row.reply_to, () => handler(msg)),
+        });
+    }
+    /** Remember who asked for `correlationId` while `work` (the responder, which replies inside it) runs. */
+    async #withReplyRoute(correlationId, replyTo, work) {
+        if (correlationId === undefined || !replyTo)
+            return work();
+        this.#replyRoutes.set(correlationId, replyTo);
         try {
-            for (const row of rows) {
-                const msg = fromJson(row.payload);
-                if (msg)
-                    handler(msg);
-                done.push(row.id);
-            }
+            await work();
         }
         finally {
-            if (done.length) {
-                await this.#client().from(TRANSPORT_TABLES.control).whereIn('id', done).delete();
-            }
+            if (this.#replyRoutes.get(correlationId) === replyTo)
+                this.#replyRoutes.delete(correlationId);
         }
-        return rows.length;
+    }
+    /**
+     * control plane → tenant: a correlated {@link RunReply}, addressed to the instance that sent the
+     * request. Only the instance handling that request knows the requester, so this must run while it
+     * is handled — which is what the responder does. A reply with no known requester is dropped and
+     * reported, never broadcast.
+     */
+    async publishRunReply(reply) {
+        const owner = this.#replyRoutes.get(reply.requestId);
+        if (owner === undefined) {
+            this.#onError(new Error(`[DbTransport] dropped the run reply for "${reply.requestId}": no requester is known for it on this instance`));
+            return;
+        }
+        if (!(await this.#messagesReady(true)))
+            return;
+        await this.#insertMessages(MESSAGE_KINDS.runReply, [{ recipient: owner, payload: reply }]);
+        this.#noteAddressed();
+    }
+    /**
+     * tenant ← control plane: consume the replies addressed to this instance. Idempotent — a second call
+     * is a no-op (one gateway per transport matches every reply by id). Consumed even while consumers are
+     * deferred: nobody else may take them.
+     */
+    onRunReply(handler) {
+        if (this.#runReplyHandler)
+            return;
+        this.#runReplyHandler = handler;
+        this.#ensureRunRepliesLoop();
+    }
+    /** Start consuming this instance's replies (once) — also on the first request, so a reply is never
+     *  left behind because the gateway subscribed late. Never backs off: a reply is on the critical path
+     *  of every proxied call. */
+    #ensureRunRepliesLoop() {
+        if (this.#messageLoops.has(MESSAGE_KINDS.runReply))
+            return;
+        this.#consumeInbox();
+        this.#startMessageLoop(MESSAGE_KINDS.runReply, {
+            shared: false,
+            recipients: () => [this.#instanceId],
+            idleBackoffMax: 1,
+            each: async (reply) => {
+                try {
+                    this.#runReplyHandler?.(reply);
+                }
+                catch (err) {
+                    this.#onError(err); // a throwing listener must not re-deliver the reply
+                }
+            },
+        });
+    }
+    /**
+     * control plane → tenant: one row of the {@link TenantEvent} per instance with a live subscription to
+     * `evt.tenant` (none: nothing is written), in one INSERT. Publishes are serialized, so each subscriber
+     * receives one publisher's events in publish order.
+     */
+    publishTenantEvent(evt) {
+        const run = this.#tenantPublishChain.then(async () => {
+            const owners = await this.#inboxes.subscribersOf(evt.tenant);
+            if (owners.length === 0)
+                return;
+            if (!(await this.#messagesReady(true)))
+                return;
+            await this.#insertMessages(MESSAGE_KINDS.tenantEvent, owners.map((recipient) => ({ recipient, payload: evt })));
+            this.#noteAddressed();
+        });
+        this.#tenantPublishChain = run.catch(() => undefined);
+        return run;
+    }
+    /**
+     * tenant ← control plane: subscribe to `tenant`'s events. Writes this instance's subscriber key
+     * (kept live every third of {@link DbTransportOptions.inboxTtlMs}) and consumes the events addressed
+     * to it — never deferred, nobody else may take them. Returns an unsubscribe fn that detaches only
+     * this handler; the last one removes the subscription.
+     */
+    onTenantEvent(tenant, handler) {
+        let handlers = this.#tenantEventHandlers.get(tenant);
+        if (!handlers) {
+            handlers = new Set();
+            this.#tenantEventHandlers.set(tenant, handlers);
+        }
+        handlers.add(handler);
+        void this.#inboxes.subscribe(tenant);
+        // Keeps running after the last unsubscribe, so events written before the key went away are
+        // consumed rather than left behind.
+        this.#startMessageLoop(MESSAGE_KINDS.tenantEvent, {
+            shared: false,
+            recipients: () => [this.#instanceId],
+            idleBackoffMax: this.#idleBackoffMax,
+            each: async (evt) => {
+                for (const h of this.#tenantEventHandlers.get(evt.tenant) ?? []) {
+                    try {
+                        h(evt);
+                    }
+                    catch (err) {
+                        this.#onError(err); // a throwing listener must not re-deliver (and reorder) the event
+                    }
+                }
+            },
+        });
+        return () => {
+            const set = this.#tenantEventHandlers.get(tenant);
+            if (!set?.delete(handler) || set.size > 0)
+                return;
+            this.#tenantEventHandlers.delete(tenant);
+            this.#inboxes.unsubscribe(tenant);
+        };
+    }
+    /** On close: remove this instance's inbox and subscriber keys, then delete what is still addressed
+     *  to it — replies, events and decisions for a process going away have no one else to go to. */
+    async #closeInboxes() {
+        await this.#inboxes.close();
+        const own = [MESSAGE_KINDS.decision, MESSAGE_KINDS.runReply, MESSAGE_KINDS.tenantEvent].filter((kind) => this.#messageLoops.has(kind));
+        if (own.length === 0)
+            return;
+        await this.#client()
+            .knexQuery()
+            .from(TRANSPORT_TABLES.messages)
+            .where('namespace', this.#namespaceValue())
+            .where('recipient', this.#instanceId)
+            .whereIn('kind', own)
+            .delete()
+            .catch((err) => this.#onError(err));
     }
     // ───────────────────────────────────────────────────────────────────────────
     // atomic claim — portable compare-and-set (no FOR UPDATE SKIP LOCKED)
     // ───────────────────────────────────────────────────────────────────────────
     /**
-     * Atomically claim up to `batchSize` un-leased rows (optionally narrowed by `where`) and stamp them
-     * with this instance's id + a fresh per-round claim timestamp, then SELECT exactly the rows this
-     * round stamped. The single conditional `UPDATE … WHERE claimed_at IS NULL OR claimed_at < stale`
-     * is the concurrency guard: two instances racing the same rows can't both stamp them (the loser's
-     * UPDATE matches nothing for those rows). A crashed claimer's rows are reclaimed once their lease
-     * expires (`claimed_at < now - leaseMs`).
+     * Atomically claim up to `limit` un-leased rows (optionally narrowed to the routing tokens `groups`)
+     * and stamp them with this instance's id + a fresh per-round claim timestamp, then SELECT exactly the
+     * rows this round stamped. The conditional `UPDATE … WHERE claimed_at IS NULL` /
+     * `… WHERE claimed_at < stale` is the concurrency guard: two instances racing the same rows can't
+     * both stamp them (the loser's UPDATE matches nothing for those rows). A crashed claimer's rows are
+     * reclaimed once their lease expires (`claimed_at < now - leaseMs`) — and with `countReclaims`, each
+     * such takeover bumps the row's `reclaims` count (the poison-task bound).
      *
-     * `claimed_at` is set to a unique, strictly-increasing-per-round value so the SELECT-back is exact
-     * even when the same instance claims several rounds in quick succession.
+     * Fresh and stale candidates are stamped by separate UPDATEs so the increment never depends on how a
+     * dialect orders the `SET` list (MySQL evaluates it left to right against the NEW values).
      */
-    async #claim(table, groups, idCol = 'step_id') {
+    async #claim(table, limit, groups, idCol = 'step_id', { countReclaims = false, byPriority = false, scope, } = {}) {
         await this.#ensureSchema();
         // The raw Knex builder — `onConflict`, the `where(cb)` grouping and `whereIn` are cleaner there
         // than through Lucid's stricter query-builder typings, and we only need plain SQL here.
-        const knex = this.#client().knexQuery();
+        const client = this.#client();
         const stale = this.#now() - this.#leaseMs;
         // Every claim is namespace-scoped: this instance only ever sees rows stamped with its own
         // namespace, so two engines/workers on different namespaces sharing one table set never
@@ -381,32 +1215,61 @@ export class DbTransport {
         // the same instance).
         const token = `${this.#instanceId}:${process.hrtime.bigint().toString(36)}`;
         const at = this.#now();
-        // 1) Pick candidate ids (unclaimed or stale-lease), oldest first, bounded by batchSize.
-        const candidates = (await knex
+        // 1) Pick candidate ids (unclaimed or stale-lease) — highest priority first when the table has
+        //    one, then oldest — bounded by `limit`.
+        const candidates = (await client
+            .knexQuery()
             .from(table)
-            .select(idCol)
+            .select(idCol, 'claimed_at')
             .where('namespace', namespace)
             .modify((q) => {
             if (groups !== undefined)
                 q.whereIn('grp', groups);
+            if (scope !== undefined)
+                q.where('kind', scope.kind).whereIn('recipient', scope.recipients);
         })
             .where((q) => q.whereNull('claimed_at').orWhere('claimed_at', '<', stale))
+            .modify((q) => {
+            if (byPriority)
+                q.orderBy('priority', 'desc');
+        })
             .orderBy(idCol === 'id' ? 'id' : 'created_at', 'asc')
-            .limit(this.#batchSize));
+            .limit(limit));
         if (candidates.length === 0)
             return { rows: [], token };
-        const ids = candidates.map((c) => c[idCol]);
+        const fresh = [];
+        const lapsed = [];
+        for (const c of candidates) {
+            (c.claimed_at == null ? fresh : lapsed).push(c[idCol]);
+        }
         // 2) Conditionally claim them — only rows STILL un-leased flip to this round's token. A racing
         //    instance's UPDATE matches nothing for the rows it lost, so no row is claimed twice.
-        await this.#client()
-            .knexQuery()
-            .from(table)
-            .where('namespace', namespace)
-            .whereIn(idCol, ids)
-            .where((q) => q.whereNull('claimed_at').orWhere('claimed_at', '<', stale))
-            .update({ claimed_by: token, claimed_at: at });
+        if (fresh.length) {
+            await client
+                .knexQuery()
+                .from(table)
+                .where('namespace', namespace)
+                .whereIn(idCol, fresh)
+                .whereNull('claimed_at')
+                .update({ claimed_by: token, claimed_at: at });
+        }
+        if (lapsed.length) {
+            await client
+                .knexQuery()
+                .from(table)
+                .where('namespace', namespace)
+                .whereIn(idCol, lapsed)
+                .where('claimed_at', '<', stale)
+                .update({
+                claimed_by: token,
+                claimed_at: at,
+                ...(countReclaims
+                    ? { reclaims: client.getWriteClient().raw('?? + 1', ['reclaims']) }
+                    : {}),
+            });
+        }
         // 3) SELECT back exactly the rows this round won.
-        const rows = (await this.#client()
+        const rows = (await client
             .knexQuery()
             .from(table)
             .select('*')
@@ -414,15 +1277,238 @@ export class DbTransport {
             .where('claimed_at', at));
         return { rows, token };
     }
+    /**
+     * Whether the tasks table has `column` — `autoCreate` adds both, but a table from an older migration
+     * run with `autoCreate: false` may lack them. Without `reclaims` the poison bound is off; without
+     * `priority` dispatch priority is ignored and rows are claimed oldest first. Either is said once
+     * through `onError`, and everything else behaves as before.
+     */
+    #hasTaskColumn(column) {
+        let known = this.#taskColumns.get(column);
+        if (!known) {
+            known = this.#ensureSchema()
+                .then(() => this.#client().schema.hasColumn(TRANSPORT_TABLES.tasks, column))
+                .then((has) => {
+                if (!has) {
+                    this.#onError(new Error(`[DbTransport] ${TRANSPORT_TABLES.tasks} has no "${column}" column, so ${MISSING_COLUMN[column].effect}. Add it — ${MISSING_COLUMN[column].ddl} — or enable autoCreate`));
+                }
+                return has;
+            })
+                .catch((err) => {
+                // Unknown yet (a transient error): go without it this time, ask again next time.
+                this.#taskColumns.delete(column);
+                this.#onError(err);
+                return false;
+            });
+            this.#taskColumns.set(column, known);
+        }
+        return known;
+    }
+    // ───────────────────────────────────────────────────────────────────────────
+    // claim renewal — every claim held, not only the row being run
+    // ───────────────────────────────────────────────────────────────────────────
+    /** Start renewing `ids` of `table`, claimed under `token`. */
+    #hold(table, token, ids) {
+        let byToken = this.#held.get(table);
+        if (!byToken) {
+            byToken = new Map();
+            this.#held.set(table, byToken);
+        }
+        byToken.set(token, new Set(ids));
+        if (this.#renewTimer)
+            return;
+        const intervalMs = Math.max(10, Math.floor(this.#leaseMs / 3));
+        this.#renewTimer = setInterval(() => {
+            if (this.#renewal)
+                return;
+            const round = this.#renewHeld()
+                .catch((err) => this.#onError(err))
+                .finally(() => {
+                if (this.#renewal === round)
+                    this.#renewal = undefined;
+            });
+            this.#renewal = round;
+        }, intervalMs);
+        this.#renewTimer.unref?.();
+    }
+    /** Stop renewing one row (settled, or no longer ours). Idempotent. */
+    #release(table, token, id) {
+        const ids = this.#held.get(table)?.get(token);
+        if (!ids)
+            return;
+        ids.delete(id);
+        if (ids.size === 0)
+            this.#releaseToken(table, token);
+    }
+    /** Stop renewing every row of a claim round. Stops the renewal timer once nothing is held. */
+    #releaseToken(table, token) {
+        const byToken = this.#held.get(table);
+        byToken?.delete(token);
+        if (byToken?.size === 0)
+            this.#held.delete(table);
+        if (this.#held.size === 0 && this.#renewTimer) {
+            clearInterval(this.#renewTimer);
+            this.#renewTimer = undefined;
+        }
+    }
+    /**
+     * Hand `tokens`' rows of `table` back: clear `claimed_by`/`claimed_at` where the claim is still ours,
+     * so the next poll anywhere claims them as fresh rows — NOT as lapsed claims, so a hand-back never
+     * counts toward the poison bound — and stop renewing them.
+     */
+    async #handBack(table, tokens) {
+        for (const token of tokens) {
+            this.#handedBack.add(token);
+            this.#releaseToken(table, token);
+        }
+        await this.#client()
+            .knexQuery()
+            .from(table)
+            .whereIn('claimed_by', tokens)
+            .update({ claimed_by: null, claimed_at: null });
+    }
+    /**
+     * Re-stamp `claimed_at` on every row this instance holds — one UPDATE per table, matched on the
+     * rounds' claim tokens, every `leaseMs / 3`. A slow but alive instance therefore keeps every row it
+     * claimed (running or queued behind a slow one); only one that stopped renewing (crashed, lost the
+     * database, or blocked its event loop past the lease) has its rows reclaimed. Fewer rows renewed
+     * than still held means exactly that happened — reported, since the step may now run twice.
+     */
+    async #renewHeld() {
+        for (const [table, byToken] of [...this.#held]) {
+            const tokens = [...byToken.keys()];
+            if (tokens.length === 0)
+                continue;
+            const renewed = await this.#client()
+                .knexQuery()
+                .from(table)
+                .whereIn('claimed_by', tokens)
+                .update({ claimed_at: this.#now() });
+            // Rows settled during the round-trip were released, not lost: count only what is still held.
+            let stillHeld = 0;
+            for (const token of tokens)
+                stillHeld += this.#held.get(table)?.get(token)?.size ?? 0;
+            if (Number(renewed) < stillHeld) {
+                this.#onError(new Error(`[DbTransport] lost ${stillHeld - Number(renewed)} of ${stillHeld} claim(s) on ${table} before renewing them — another instance reclaimed them and may process them too; raise leaseMs if this process blocks its event loop`));
+            }
+        }
+    }
+    // ───────────────────────────────────────────────────────────────────────────
+    // worker liveness + health — `${P}-worker-heartbeat:<token>:<instanceId>` in the worker-key table
+    // ───────────────────────────────────────────────────────────────────────────
+    /** The prefix the liveness keys are built from (`durable`, `durable-<namespace>`). */
+    #keyPrefix() {
+        return effectivePrefix(WORKER_KEY_PREFIX, this.#namespace);
+    }
+    /** The task loop serves `token`: run the concurrency controller and beat its liveness key. */
+    #beginServing(token) {
+        this.#controller.start();
+        this.#startWorkerHeartbeat(token);
+    }
+    /**
+     * Refresh every served routing token's liveness key on one shared interval (default 10s, live 35s),
+     * stamped with the controller's {@link WorkerStatus} snapshot — the same `{ ts, status }` value and
+     * key the BullMQ and queue transports write, so the dashboard's worker cards read all three alike.
+     * A new token beats at once. Best-effort: a failed write goes to `onError` and the key expires.
+     */
+    #startWorkerHeartbeat(token) {
+        const ttlMs = Math.max(WORKER_HEARTBEAT_TTL_SECONDS * 1000, this.#workerHeartbeatIntervalMs * 3);
+        const beatOne = (t) => {
+            // One write per token at a time: a beat due while the last one is still in flight is skipped.
+            // Beats are not queued behind a slow store — every interval used to start a write regardless,
+            // so a store slower than the interval piled up writes without bound, each holding a pool
+            // connection the reads (groupHealth, the claim loop) then waited behind.
+            if (this.#beatsInFlight.has(t))
+                return;
+            this.#beatsInFlight.add(t);
+            const now = this.#now();
+            const key = workerHeartbeatKey(this.#keyPrefix(), t, this.#instanceId);
+            const value = heartbeatKeyValue(now, this.#controller.snapshot());
+            const write = Promise.resolve()
+                .then(() => this.#workerKeys().put(key, value, now + ttlMs))
+                .catch((err) => this.#onError(err))
+                .finally(() => this.#beatsInFlight.delete(t));
+            this.#livenessWrites.add(write);
+            void write.finally(() => this.#livenessWrites.delete(write));
+        };
+        if (!this.#heartbeatTokens.has(token)) {
+            this.#heartbeatTokens.add(token);
+            beatOne(token);
+        }
+        if (this.#workerHeartbeatTimer)
+            return;
+        this.#workerHeartbeatTimer = setInterval(() => {
+            for (const t of this.#heartbeatTokens)
+                beatOne(t);
+        }, this.#workerHeartbeatIntervalMs);
+        this.#workerHeartbeatTimer.unref?.();
+    }
+    /**
+     * Per-group worker health, for `engine.workerHealth()` and the dashboard/Telescope worker panels.
+     *
+     * - `depth` is every task row of `group` in this namespace — pending AND claimed (running, or held
+     *   by a worker that died and not yet reclaimed): the DB can count both, so it is closer to BullMQ's
+     *   waiting+active than the queue transport's pending-only count.
+     * - `liveWorkers` are the unexpired `${P}-worker-heartbeat:<group>:<instance>` keys in the
+     *   worker-key table, each with the worker's {@link WorkerStatus} when it carries one (this
+     *   transport's own beats do; a `WorkerRuntime`'s compact beat reads as liveness only).
+     */
+    async groupHealth(group) {
+        await this.#ensureSchema();
+        const [counted, entries] = await Promise.all([
+            this.#client()
+                .knexQuery()
+                .from(TRANSPORT_TABLES.tasks)
+                .where('namespace', this.#namespaceValue())
+                .where('grp', group)
+                .count({ n: '*' })
+                .first(),
+            this.#liveWorkerKeys(),
+        ]);
+        const marker = workerHeartbeatKey(this.#keyPrefix(), group, '');
+        const liveWorkers = [];
+        for (const { key, value } of entries) {
+            if (!key.startsWith(marker))
+                continue;
+            const parsed = parseHeartbeatValue(value);
+            liveWorkers.push({
+                group,
+                instanceId: key.slice(marker.length),
+                lastBeatAt: parsed.lastBeatAt,
+                ...(parsed.status !== undefined
+                    ? { status: parsed.status }
+                    : {}),
+            });
+        }
+        return { group, depth: Number(counted?.n) || 0, liveWorkers };
+    }
     // ───────────────────────────────────────────────────────────────────────────
     // worker advertisement (design §7.2) — a key/value table with an expiry per key
     // ───────────────────────────────────────────────────────────────────────────
     #workerKeys() {
-        this.#workerKeyStore ??= lucidWorkerKeys(this.#db, {
-            connectionName: this.#connectionName,
-            autoCreate: this.#autoCreate,
-            now: () => this.#now(),
-        });
+        if (!this.#workerKeyStore) {
+            // The table is created with the others by #ensureSchema — waited for first, so a key written
+            // while the schema is still being created never races its CREATE.
+            const keys = lucidWorkerKeys(this.#db, {
+                connectionName: this.#connectionName,
+                autoCreate: false,
+                now: () => this.#now(),
+            });
+            this.#workerKeyStore = {
+                put: async (key, value, expiresAt) => {
+                    await this.#ensureSchema();
+                    await keys.put(key, value, expiresAt);
+                },
+                remove: async (removed) => {
+                    await this.#ensureSchema();
+                    await keys.remove(removed);
+                },
+                live: async (at) => {
+                    await this.#ensureSchema();
+                    return keys.live(at);
+                },
+            };
+        }
         return this.#workerKeyStore;
     }
     /** The writer a worker runtime's registry advertises its descriptor and heartbeat through. */
@@ -455,17 +1541,85 @@ export class DbTransport {
         this.#pollers.reopen();
         await this.#ensureSchema();
     }
-    /** Stop every poll loop and wait (bounded) for in-flight ticks — a claimed task mid-handler gets
-     *  to finish + write its result instead of being abandoned to the lease expiry. Does not close
-     *  the shared `Database`. */
+    /**
+     * Stop every poll loop and wait — bounded by {@link DbTransportOptions.closeTimeoutMs}, 5s by
+     * default — for in-flight work: the tasks in a handler get to finish and write their result, and a
+     * result batch being settled gets to finish. Whatever is still unfinished then is HANDED BACK: its
+     * claims are cleared (only where still ours) so another worker picks the rows up at once, instead of
+     * after the lease — and uncounted, so a deploy never pushes a long step toward the poison bound. A
+     * handler still running here may finish after another worker re-ran its step; its result is
+     * dropped. Does not close the shared `Database`.
+     */
     async stop() {
         this.#pollers.stopAll();
         this.#taskLoop = undefined;
-        await this.#pollers.drain(5_000);
+        this.#controller.stop();
+        if (this.#sweepTimer)
+            clearInterval(this.#sweepTimer);
+        this.#sweepTimer = undefined;
+        if (this.#workerHeartbeatTimer)
+            clearInterval(this.#workerHeartbeatTimer);
+        // A clean stop withdraws this instance's liveness keys at once, so the dashboard and
+        // `listWorkerGroups` drop it now instead of after the 35s TTL. In-flight beats land first, or one
+        // could re-create a key after its removal.
+        if (this.#heartbeatTokens.size > 0) {
+            await Promise.allSettled([...this.#livenessWrites]);
+            const keys = [...this.#heartbeatTokens].map((t) => workerHeartbeatKey(this.#keyPrefix(), t, this.#instanceId));
+            this.#heartbeatTokens.clear();
+            await Promise.resolve()
+                .then(() => this.#workerKeys().remove(keys))
+                .catch((err) => this.#onError(err));
+        }
+        this.#workerHeartbeatTimer = undefined;
+        this.#heartbeatTokens.clear();
+        const deadline = Date.now() + this.#closeTimeoutMs;
+        await this.#pollers.drain(this.#closeTimeoutMs);
+        if (this.#runningTasks.size > 0) {
+            let timer;
+            await Promise.race([
+                Promise.allSettled([...this.#runningTasks]),
+                new Promise((resolve) => {
+                    timer = setTimeout(resolve, Math.max(deadline - Date.now(), 0));
+                }),
+            ]);
+            clearTimeout(timer);
+        }
+        // A renewal round in flight would re-stamp what we are about to hand back.
+        await this.#renewal;
+        for (const [table, byToken] of [...this.#held]) {
+            const tokens = [...byToken.keys()];
+            if (tokens.length === 0)
+                continue;
+            await this.#handBack(table, tokens).catch((err) => this.#onError(err));
+        }
+        await this.#sweeping;
+        await this.#closeInboxes();
     }
     /** Stop the pollers (alias of {@link stop} for the `Transport.close` contract). */
     async close() {
         await this.stop();
     }
 }
+/**
+ * Run `fn` over `items` with at most `limit` in flight, in order of start. The first failure stops
+ * starting new items; the ones already running settle, then it rethrows.
+ */
+async function eachLimited(items, limit, fn) {
+    let next = 0;
+    let failure;
+    const lane = async () => {
+        while (!failure && next < items.length) {
+            const item = items[next++];
+            try {
+                await fn(item);
+            }
+            catch (err) {
+                failure ??= { err };
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, lane));
+    if (failure)
+        throw failure.err;
+}
 //# sourceMappingURL=db.js.map