@adonis-agora/durable 0.9.1 → 0.11.0

package/dist/src/engine.js

@@ -1,6 +1,7 @@
 import { InMemoryAdmissionBackend } from './admission.js';
 import { backoffDelay } from './backoff.js';
 import { instantCheckpoint, stepCheckpoint } from './checkpoints.js';
+import { WorkflowBlocked, controlPlaneDescriptor, planDispatch, } from './dispatch-routing.js';
 import { parseDuration } from './duration.js';
 import { Entities } from './entities.js';
 import { ContinueAsNew, NonDeterminismError, RemoteStepTimeout, WorkflowSuspended, } from './errors.js';
@@ -67,6 +68,9 @@ export class WorkflowEngine {
     namespace;
     leaseMs;
     maxRecoveryAttempts;
+    reconcileMs;
+    remoteRedispatchMs;
+    remoteRedispatchMax;
     webhookUrl;
     traceparent;
     context;
@@ -77,6 +81,11 @@ export class WorkflowEngine {
     trackStepStart;
     /** Where a freshly-started run executes — in-process by default (see {@link RunDispatcher}). */
     runDispatcher;
+    /** The control-plane's OWN handshake descriptor — negotiated against each worker's descriptor to
+     *  detect protocol incompatibility before dispatch (design §7.3/§7.4). Built once at construction. */
+    cpDescriptor;
+    /** How long a `blocked` run waits before the recovery poll re-checks the live fleet (design §7.5). */
+    blockedPollMs;
     /** Per-key serialization for singleton workflows (admission, back-pressure, notify-on-release). */
     singletons;
     /** Every registered workflow, keyed by `name@version` — so old versions stay runnable. */
@@ -128,12 +137,29 @@ export class WorkflowEngine {
         this.admission.onFreed?.((queue) => this.wakeQueueWaiters(queue));
         this.instanceId = deps.instanceId ?? globalThis.crypto.randomUUID();
         this.namespace = deps.namespace ?? 'default';
+        // The control-plane's own handshake descriptor — the side negotiated AGAINST each worker so a
+        // protocol-incompatible fleet is detected before dispatch (design §7.3). Scoped to this engine's
+        // namespace/partition; capabilities/protocol are config-driven (default: current single major).
+        this.cpDescriptor = controlPlaneDescriptor({
+            instanceId: this.instanceId,
+            ...(deps.controlPlaneCapabilities !== undefined
+                ? { capabilities: deps.controlPlaneCapabilities }
+                : {}),
+            ...(deps.controlPlaneProtocol !== undefined ? { protocol: deps.controlPlaneProtocol } : {}),
+            ...(this.namespace !== 'default' ? { namespace: this.namespace } : {}),
+        });
+        this.blockedPollMs = deps.blockedPollMs ?? 5000;
         // Propagate the engine's namespace to its transport(s) so the SAME namespace that partitions the
         // store also partitions the transport's queues/keys — set once on the engine, applied everywhere.
         // The transport makes "default" a no-op (byte-identical names); an empty pool is a no-op too.
         this.pool.useNamespace(this.namespace);
         this.leaseMs = deps.leaseMs ?? 30_000;
         this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
+        // Default 5min; an explicit 0 disables the fallback (opt back into wake-forever-on-lost-event).
+        this.reconcileMs = deps.reconcileMs === undefined ? 300_000 : deps.reconcileMs || undefined;
+        // Opt-in: unset (or 0) leaves the by-design "re-suspend a lost dispatch, never re-dispatch" behavior.
+        this.remoteRedispatchMs = deps.remoteRedispatchMs || undefined;
+        this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
         this.webhookUrl = deps.webhookUrl;
         this.traceparent = deps.traceparent;
         this.context = deps.context;
@@ -267,6 +293,7 @@ export class WorkflowEngine {
             validateInput: opts?.validateInput,
             onEvent: opts?.onEvent,
             eventBatch: opts?.eventBatch,
+            requires: opts?.requires,
         };
         this.workflows.set(versionKey(name, version), registered);
         const current = this.latest.get(name);
@@ -297,6 +324,7 @@ export class WorkflowEngine {
             executionTimeoutMs: opts.executionTimeout != null ? parseDuration(opts.executionTimeout) : undefined,
             validateInput: opts.validateInput,
             remote: { group: opts.group, executor: opts.executor },
+            requires: opts.requires,
         };
         this.workflows.set(versionKey(name, version), registered);
         const current = this.latest.get(name);
@@ -661,11 +689,16 @@ export class WorkflowEngine {
         return undefined;
     }
     /**
-     * Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
-     * boot. A run still not due re-suspends cheaply without running new work.
+     * Resume every suspended run whose durable timer is due — AND every run parked `blocked` (no
+     * capable/compatible worker, design §7.5) whose recovery poll is due. Call periodically (a poller)
+     * and on boot. A blocked run re-drives to re-check the live fleet: it dispatches the moment a
+     * capable+compatible worker has appeared, else re-parks `blocked` with a fresh `wakeAt`. A run still
+     * not due re-suspends cheaply without running new work.
      */
     async resumeDueTimers(nowMs = this.clock()) {
-        return this.resumeLeased(await this.store.listDueTimers(nowMs, this.namespace), nowMs);
+        const due = await this.store.listDueTimers(nowMs, this.namespace);
+        const blocked = await this.dueBlockedRuns(nowMs);
+        return this.resumeLeased([...due, ...blocked], nowMs);
     }
     /**
      * Lease and execute one run by id — the worker side of dispatch. Acquires the recovery lease (so
@@ -695,15 +728,124 @@ export class WorkflowEngine {
      * inline — or null if the run is unknown. The dashboard "retry" goes through here so it can't block
      * the HTTP request on workflow execution.
      */
-    async requeue(runId) {
+    async requeue(runId, _seen) {
+        const seen = _seen ?? new Set();
+        if (seen.has(runId))
+            return null;
+        seen.add(runId);
         const run = await this.store.getRun(runId);
         if (!run)
             return null;
+        // A FAILED/DEAD run's replay deterministically re-throws its recorded failures — an exhausted
+        // failed checkpoint rethrows, and an awaited child's failure completion rethrows through
+        // unwrapCompletion — so "retry" without resetting that state is a no-op that re-fails in
+        // milliseconds. Reset the failure state first so replay RE-EXECUTES the failed parts:
+        // - a `failed` checkpoint becomes retryable-now (attempts 0, wake immediately, `retryable`
+        //   forced back on): the durable-retry machinery re-dispatches it as a fresh first attempt;
+        // - an awaited-child `signal:child:` checkpoint holding a FAILURE completion returns to its
+        //   live `running` placeholder, so replay re-registers the `child:<id>` waiter — a separately
+        //   retried child's completion (already buffered, or still to come) then resumes this run.
+        //   Retry child and parent in either order; signal buffering makes it converge.
+        if (run.status === 'failed' || run.status === 'dead') {
+            const isFailureCompletion = (v) => typeof v === 'object' && v !== null && v.ok === false;
+            for (const cp of await this.store.listCheckpoints(runId)) {
+                if (cp.status === 'failed') {
+                    await this.store.saveCheckpoint({
+                        ...cp,
+                        attempts: 0,
+                        wakeAt: this.clock(),
+                        ...(cp.error ? { error: { ...cp.error, retryable: true } } : {}),
+                    });
+                }
+                else if (cp.kind === 'signal' &&
+                    cp.name.startsWith('signal:child:') &&
+                    cp.status === 'completed' &&
+                    isFailureCompletion(cp.output)) {
+                    await this.store.saveCheckpoint({ ...cp, status: 'running', output: undefined });
+                    // CASCADE: retrying only the parent is useless in a live engine — replay re-registers
+                    // the `child:<id>` waiter and the reconciler re-delivers the child's still-FAILED
+                    // terminal state within seconds, re-failing the parent with the same error. Requeue the
+                    // failed child too so the pair converges: whichever finishes the handshake last (waiter
+                    // registered vs completion buffered) resumes the parent.
+                    //
+                    // Exception: a SUCCESS already buffered on this token (a completed `~retry~` fix of the
+                    // child — see notifyParent's origin delivery) is the outcome the replay will consume, so
+                    // re-running the failed origin would be pure waste. Peek = take + re-buffer.
+                    const childId = cp.name.slice('signal:child:'.length);
+                    // Drain-and-restore keeps relative order (a partial take+re-buffer would rotate it).
+                    const parked = [];
+                    for (;;) {
+                        const buffered = await this.store.takeBufferedSignal(`child:${childId}`);
+                        if (!buffered)
+                            break;
+                        parked.push(buffered.payload);
+                    }
+                    for (const payload of parked)
+                        await this.store.bufferSignal(`child:${childId}`, payload);
+                    const bufferedSuccess = parked.some((payload) => typeof payload === 'object' &&
+                        payload !== null &&
+                        payload.ok === true);
+                    if (!bufferedSuccess) {
+                        const child = await this.store.getRun(childId);
+                        if (child && (child.status === 'failed' || child.status === 'dead')) {
+                            await this.requeue(childId, seen);
+                        }
+                    }
+                }
+            }
+        }
         await this.store.releaseRunLock(runId);
-        await this.store.updateRun(runId, { status: 'pending', updatedAt: new Date() });
+        // Explicit `error: undefined` CLEARS the stale failure (the store spreads the patch over the
+        // run) — otherwise the dashboard keeps showing the old error while the run is re-executing.
+        await this.store.updateRun(runId, {
+            status: 'pending',
+            error: undefined,
+            updatedAt: new Date(),
+        });
         await this.runDispatcher.dispatch(runId);
         return { runId, status: 'pending' };
     }
+    /**
+     * Explicitly re-dispatch every remote step of `runId` stuck `pending` — the OPERATOR escape hatch for
+     * a run whose dispatched step job was LOST (worker crashed with no result, or a transport that dropped
+     * the job). No automatic recovery re-drives these: a reconcile re-drive re-suspends a still-pending
+     * step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete` only reclaims LEASED runs,
+     * and `requeue` just replays back to the pending guard. This re-enqueues the same `stepId` so the
+     * (idempotent) step re-runs and its result resumes the run. Safe to call on a healthy run — it re-runs
+     * only in-flight `pending` remote steps, which are idempotent by the durable contract. Returns the
+     * run's current status (with the count re-dispatched), or null if the run is unknown.
+     */
+    async redispatchPending(runId) {
+        const run = await this.store.getRun(runId);
+        if (!run)
+            return null;
+        const pending = (await this.store.listCheckpoints(runId)).filter((cp) => cp.kind === 'remote' && cp.status === 'pending');
+        let redispatched = 0;
+        for (const cp of pending) {
+            if (!cp.stepId)
+                continue;
+            const reAttempt = cp.attempts + 1;
+            const reEnqueuedAt = new Date();
+            await this.store.saveCheckpoint({
+                ...cp,
+                attempts: reAttempt,
+                enqueuedAt: reEnqueuedAt,
+                startedAt: reEnqueuedAt,
+                finishedAt: reEnqueuedAt,
+            });
+            await this.dispatchRemoteTask({
+                runId,
+                seq: cp.seq,
+                name: cp.name,
+                stepId: cp.stepId,
+                group: cp.workerGroup ?? sanitizeQueueToken(cp.name),
+                input: cp.input,
+                attempt: reAttempt,
+            });
+            redispatched += 1;
+        }
+        return { runId, status: run.status, redispatched };
+    }
     /**
      * **Fix-and-replay**: re-run a run (typically a `dead`/`failed` one) with a corrected `input`, as a
      * fresh run with clean history. It's a NEW run — `newRunId` defaults to `<runId>~retry~<uuid>` — so
@@ -982,6 +1124,15 @@ export class WorkflowEngine {
      */
     notifyParent(runId, completion) {
         void this.signal(`child:${runId}`, completion).catch(() => undefined);
+        // A fix-and-replay run (`<origin>~retry~<hash>`) is standalone, but its SUCCESS is the
+        // origin's outcome for all practical purposes: deliver it on the ORIGIN's token too, so a
+        // parent that failed on that child and is retried later consumes this success (buffered or
+        // live) instead of waiting on a child nobody re-runs. Failures stay retry-only — a failed
+        // fix attempt must not poison the origin's token.
+        const at = runId.lastIndexOf('~retry~');
+        if (at !== -1 && completion.ok) {
+            void this.signal(`child:${runId.slice(0, at)}`, completion).catch(() => undefined);
+        }
     }
     /**
      * Deferred child start shared by the in-process ctx host and the remote `startChild` command.
@@ -1393,7 +1544,11 @@ export class WorkflowEngine {
         if (latest?.status === 'cancelled') {
             return { runId: run.id, status: 'cancelled', error: latest.error };
         }
-        await this.store.updateRun(run.id, { status: 'suspended', wakeAt: outcome.wakeAt, updatedAt });
+        await this.store.updateRun(run.id, {
+            status: 'suspended',
+            wakeAt: this.reconcileWakeAt(outcome.wakeAt),
+            updatedAt,
+        });
         this.emit({
             type: 'run.suspended',
             runId: run.id,
@@ -1402,6 +1557,17 @@ export class WorkflowEngine {
         });
         return { runId: run.id, status: 'suspended' };
     }
+    /**
+     * A wakeAt to persist on a suspend: the natural timer if the turn produced one (a `ctx.sleep`, a
+     * remote-step deadline), else the {@link reconcileMs} fallback so an event-waiting run (child/signal/
+     * timeout-less remote step) can't be orphaned forever if its wake is lost. `undefined` only when the
+     * fallback is disabled (`reconcileMs: 0`). See {@link WorkflowEngineDeps.reconcileMs}.
+     */
+    reconcileWakeAt(wakeAt) {
+        if (wakeAt != null)
+            return wakeAt;
+        return this.reconcileMs != null ? this.clock() + this.reconcileMs : undefined;
+    }
     async runRemoteExecution(run, registered) {
         const remote = registered.remote;
         if (run.status === 'pending') {
@@ -1426,6 +1592,12 @@ export class WorkflowEngine {
             await this.store.releaseRunLock(run.id);
             return { runId: run.id, status: 'suspended' };
         }
+        // Capability/protocol guard (design §7.5): a remote workflow's turn dispatches a WorkflowTask to
+        // its group — if no live worker on that group can run it, park `blocked` before advancing rather
+        // than handing the task to the executor to enqueue into a void. Re-checked each turn (recovery).
+        const routePlan = await this.planRoute(remote.group, registered.requires);
+        if (routePlan?.status === 'blocked')
+            return this.parkBlocked(run, routePlan);
         const history = await this.remoteHistory(run.id);
         let decision;
         try {
@@ -1757,6 +1929,12 @@ export class WorkflowEngine {
                 queueMicrotask(() => void this.start(run.workflow, err.input, nextId).catch(() => undefined));
                 return { runId: run.id, status: 'completed' };
             }
+            if (err instanceof WorkflowBlocked) {
+                // No live worker can run the next dispatch (capability/protocol) — park `blocked` with the
+                // structured reason + diagnostics instead of suspending on a timer that would re-drive into the
+                // same void. The blocked-recovery poll re-drives it when a capable+compatible worker appears.
+                return this.parkBlocked(run, err.plan);
+            }
             if (err instanceof WorkflowSuspended) {
                 // A compensating cancel resumed this run to reach here: the replay re-registered the saga,
                 // so undo the completed steps in reverse and mark it cancelled instead of re-suspending.
@@ -1899,6 +2077,12 @@ export class WorkflowEngine {
             startChild: (workflow, input, id, priority) => {
                 this.startChildDeferred(workflow, input, id, { priority });
             },
+            // Deferred for the same reentrancy reason as `startChild` above. `cancel()` is already
+            // idempotent on a terminal/cancelled run (returns its existing status without side effects), so
+            // no extra guard is needed here for the failFast replay case (re-issuing the same cancel calls).
+            cancelChild: (childId) => {
+                queueMicrotask(() => void this.cancel(childId).catch(() => undefined));
+            },
             // Shallow-merge into the run's searchAttributes (the ctx primitive makes this exactly-once).
             upsertSearchAttributes: async (runId, attrs) => {
                 const run = await this.store.getRun(runId);
@@ -1978,6 +2162,89 @@ export class WorkflowEngine {
         const result = await this.signal(`update:${runId}:${name}`, arg);
         return { accepted: true, run: result };
     }
+    // ---------------------------------------------------------------------------
+    // Capability- & protocol-aware dispatch guard (design §7.5/§7.6)
+    // ---------------------------------------------------------------------------
+    /** The LIVE worker handshake descriptors advertised on `token`, read from the primary transport's
+     *  descriptor keyspace. Empty when no transport can introspect it (in-process / pre-handshake broker)
+     *  OR nobody is publishing — either way the guard is skipped and dispatch proceeds legacy
+     *  assume-compatible (design §7.7). Never throws (the transport degrades to `[]`). */
+    async liveDescriptorsFor(token) {
+        const read = this.primaryTransport?.listWorkerDescriptors;
+        if (!read)
+            return [];
+        try {
+            return await read.call(this.primaryTransport, token);
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Plan a dispatch to `token` for a handler requiring `requires` (design §7.5). Returns:
+     * - `undefined` — NO live descriptors published on this token: the guard does not engage, so a
+     *   pre-handshake fleet (or an in-process transport) keeps flowing exactly as before (design §7.7);
+     * - a `routable` plan — ≥1 live worker is capability-capable AND protocol-compatible (dispatch);
+     * - a `blocked` plan — descriptors exist but none can run it (park the run, never dispatch into it).
+     */
+    async planRoute(token, requires) {
+        const descriptors = await this.liveDescriptorsFor(token);
+        if (descriptors.length === 0)
+            return undefined; // legacy/no-runtime path — skip the guard
+        return planDispatch(requires ?? [], descriptors, this.cpDescriptor, token);
+    }
+    /** Throw {@link WorkflowBlocked} when no live worker can run a dispatch to `token` — so the caller's
+     *  run-execution catch parks the run `blocked` (with the plan) instead of persisting a `pending`
+     *  checkpoint + dispatching into a queue nobody consumes (design §7.5). A no-op when the guard does
+     *  not engage (no descriptors) or a capable+compatible worker exists. */
+    async ensureRoutable(token, requires) {
+        const plan = await this.planRoute(token, requires);
+        if (plan?.status === 'blocked')
+            throw new WorkflowBlocked(plan);
+    }
+    /**
+     * Park a run `blocked`: no live worker can run its next dispatch (design §7.5/§7.6). Persists the
+     * `blocked` status with a human reason + a `wakeAt` so the blocked-recovery poll re-drives it, emits
+     * the LOUD structured `capability.unavailable`/`protocol.incompatible` diagnostics event (carrying
+     * both descriptors + the precise delta), and releases the run lease so any instance can re-drive it.
+     * Re-checks the persisted status first so a concurrent `cancel` is never clobbered.
+     */
+    async parkBlocked(run, blocked) {
+        const latest = await this.store.getRun(run.id);
+        if (latest && (latest.status === 'cancelled' || latest.status === 'completed')) {
+            await this.store.releaseRunLock(run.id);
+            return { runId: run.id, status: latest.status, output: latest.output, error: latest.error };
+        }
+        const error = { message: blocked.reason, retryable: true };
+        const wakeAt = this.clock() + this.blockedPollMs;
+        await this.store.updateRun(run.id, {
+            status: 'blocked',
+            error,
+            wakeAt,
+            updatedAt: new Date(),
+        });
+        this.emit({
+            type: blocked.code,
+            runId: run.id,
+            workflow: run.workflow,
+            namespace: run.namespace,
+            error,
+            diagnostics: blocked.diagnostics,
+        });
+        // Release the lease so the blocked-recovery poll (this or any instance) can re-drive it promptly
+        // once a capable+compatible worker appears — mirrors the suspended path's release.
+        await this.store.releaseRunLock(run.id);
+        return { runId: run.id, status: 'blocked', error };
+    }
+    /** Runs parked `blocked` whose recovery poll is due (`wakeAt <= nowMs`), namespace-scoped like
+     *  {@link resumeDueTimers}. Re-driving one re-checks the live fleet: it dispatches if a
+     *  capable+compatible worker has appeared, else re-parks with a fresh `wakeAt` (design §7.5). */
+    async dueBlockedRuns(nowMs) {
+        const blocked = await this.store.listRuns({ statuses: ['blocked'] });
+        return blocked.filter((r) => (r.namespace === undefined || r.namespace === this.namespace) &&
+            r.wakeAt !== undefined &&
+            r.wakeAt <= nowMs);
+    }
     async callRemote(runId, seq, step, input, queue, transport, replay, admission) {
         // Read the prefix from the per-execution snapshot (avoids the O(N²) replay SELECTs); a seq absent
         // from the snapshot — not yet dispatched, or written after the snapshot — falls back to the store.
@@ -1995,8 +2262,57 @@ export class WorkflowEngine {
         // pod can scale down or crash mid-step without losing the run or re-running completed work.
         if (step.timeoutMs)
             return this.callRemoteInMemory(runId, seq, step, input, transport);
-        if (existing?.status === 'pending')
-            throw new WorkflowSuspended(); // dispatched; keep waiting
+        if (existing?.status === 'pending') {
+            // Dispatched; normally we just keep waiting for the result to resume the run. But a LOST dispatch
+            // (worker crashed with no result, or the transport dropped the job) would hang here forever — a
+            // reconcile re-drive replays straight back to this guard. Opt-in self-heal: once the step has been
+            // pending longer than `remoteRedispatchMs`, re-dispatch the same stepId (the idempotent step
+            // re-runs, its result resumes the run), bounded by `remoteRedispatchMax` so a never-settling step
+            // fails instead of looping. Unset (default) keeps the by-design "re-suspend, never re-dispatch".
+            if (this.remoteRedispatchMs == null)
+                throw new WorkflowSuspended();
+            // Stamp a redispatch deadline (clock-space, persisted) the first time we see this pending step,
+            // stable across replays and crashes — mirrors the failed-retry backoff below. The run's wakeAt
+            // becomes this deadline, so a reconcile re-drive lands exactly when it's due to re-dispatch.
+            if (existing.wakeAt == null) {
+                const wakeAt = this.clock() + this.remoteRedispatchMs;
+                await this.store.saveCheckpoint({ ...existing, wakeAt });
+                throw new WorkflowSuspended(wakeAt);
+            }
+            if (this.clock() < existing.wakeAt)
+                throw new WorkflowSuspended(existing.wakeAt);
+            // Past the deadline with no result — the dispatch is presumed lost. Re-dispatch (bounded by
+            // `remoteRedispatchMax` so a step that never settles fails the run instead of looping forever).
+            if (existing.attempts >= this.remoteRedispatchMax) {
+                throw new RemoteStepError({
+                    message: `remote step "${step.name}" lost — no result after ${existing.attempts} re-dispatch(es)`,
+                    code: 'remote_step_lost',
+                });
+            }
+            const reAttempt = existing.attempts + 1;
+            const nextDeadline = this.clock() + this.remoteRedispatchMs;
+            const reEnqueuedAt = new Date();
+            await this.store.saveCheckpoint({
+                ...existing,
+                attempts: reAttempt,
+                wakeAt: nextDeadline,
+                enqueuedAt: reEnqueuedAt,
+                startedAt: reEnqueuedAt,
+                finishedAt: reEnqueuedAt,
+            });
+            await this.dispatchRemoteTask({
+                runId,
+                seq,
+                name: step.name,
+                stepId: existing.stepId ?? stepId(runId, seq),
+                group: existing.workerGroup ?? tenantGroup(sanitizeQueueToken(step.name), step.partition),
+                input: existing.input,
+                priority: admission?.priority,
+                attempt: reAttempt,
+                transport,
+            });
+            throw new WorkflowSuspended(nextDeadline);
+        }
         // Durable retry: a failed attempt re-dispatches up to `retries`, spacing attempts by `backoff` —
         // unless the worker marked the error non-retryable (a deterministic verdict like a declined card).
         // The retry deadline is stamped on the failed checkpoint as `wakeAt` (clock-space, persisted) the
@@ -2047,6 +2363,11 @@ export class WorkflowEngine {
         // used at BOTH the checkpoint's workerGroup and the dispatched task's routing `group`, so the same
         // token the worker subscribes to per handler name serves this step.
         const token = tenantGroup(sanitizeQueueToken(step.name), step.partition);
+        // Capability/protocol guard (design §7.5): if the live fleet on this token can't run the step,
+        // throw WorkflowBlocked BEFORE the pending checkpoint + dispatch — so the run parks `blocked`
+        // (recovered by the blocked poll) instead of writing a `pending` row a `WorkflowSuspended` re-drive
+        // would mistake for "already dispatched" and wait on forever. A no-op when no descriptors exist.
+        await this.ensureRoutable(token, step.requires);
         const enqueuedAt = new Date();
         // Persist the pending checkpoint BEFORE dispatching, so a fast result always finds it to complete.
         await this.store.saveCheckpoint({
@@ -2063,21 +2384,39 @@ export class WorkflowEngine {
             startedAt: enqueuedAt, // placeholders until the worker result lands
             finishedAt: enqueuedAt,
         });
-        await this.pool.dispatch({
+        await this.dispatchRemoteTask({
             runId,
             seq,
             name: step.name,
             stepId: id,
             group: token,
             input: validInput,
-            traceparent: this.traceparent?.(),
-            context: this.context?.(),
             priority: admission?.priority,
             attempt,
-        }, transport);
+            transport,
+        });
         this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
         throw new WorkflowSuspended();
     }
+    /**
+     * Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
+     * dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
+     * {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable.
+     */
+    dispatchRemoteTask(task) {
+        return this.pool.dispatch({
+            runId: task.runId,
+            seq: task.seq,
+            name: task.name,
+            stepId: task.stepId,
+            group: task.group,
+            input: task.input,
+            traceparent: this.traceparent?.(),
+            context: this.context?.(),
+            priority: task.priority,
+            attempt: task.attempt,
+        }, task.transport);
+    }
     /**
      * Complete a durable remote step from its worker result and resume the run — runs on whichever
      * instance receives the result (the dispatching one may be gone), so the run is crash/scale-safe.
@@ -2157,6 +2496,9 @@ export class WorkflowEngine {
             throw new Error('remote steps require a Transport');
         const validInput = step.input ? step.input.parse(input) : input;
         const token = tenantGroup(sanitizeQueueToken(step.name), step.partition);
+        // Capability/protocol guard (design §7.5) — same as the durable path, before the `step.started`
+        // emit + dispatch, so a step nobody can run parks `blocked` instead of awaiting a phantom worker.
+        await this.ensureRoutable(token, step.requires);
         const id = stepId(runId, seq);
         const enqueuedAt = new Date();
         this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });