@adonis-agora/durable 0.40.0 → 0.41.1

package/dist/src/engine.js

@@ -1,6 +1,6 @@
 import { InMemoryAdmissionBackend } from './admission.js';
 import { backoffDelay, MAX_BACKOFF_MS } from './backoff.js';
-import { instantCheckpoint, stepCheckpoint } from './checkpoints.js';
+import { instantCheckpoint, lostStepEvent, stepCheckpoint } from './checkpoints.js';
 import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispatch-routing.js';
 import { parseDuration } from './duration.js';
 import { Entities } from './entities.js';
@@ -305,9 +305,16 @@ export class WorkflowEngine {
             await this.completeRemoteResult(result);
         }, 
         // A heartbeat for an in-flight long step resets its liveness window (see callRemote) and is
-        // persisted (throttled) on the step's checkpoint so liveness is visible cross-process.
+        // persisted (throttled) on the step's checkpoint so liveness is visible cross-process. With no
+        // in-memory waiter (the step suspended DURABLY, possibly on another instance) the beat rearms
+        // the step's durable LEASE instead, so a worker still holding a long step is never presumed
+        // lost and double-dispatched (see rearmStepLease).
         async (beat) => {
-            this.heartbeatResets.get(beat.stepId)?.();
+            const reset = this.heartbeatResets.get(beat.stepId);
+            if (reset)
+                reset();
+            else
+                await this.rearmStepLease(beat.runId, beat.seq);
             this.persistHeartbeat(beat);
         }, 
         // A remote workflow worker streams each local step's lifecycle (running → completed/failed) so
@@ -1270,14 +1277,30 @@ export class WorkflowEngine {
             }
             return (await this.store.listIncompleteRuns(this.namespace)).filter((r) => r.lockedUntil === undefined || r.lockedUntil <= nowMs);
         });
-        for (const run of incomplete) {
+        for (const listed of incomplete) {
             // A live worker renews its lease, so an acquirable lease means the run is genuinely orphaned
             // (its worker crashed). Skip the ones still owned.
-            const acquired = await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs);
+            const acquired = await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
             if (!acquired)
                 continue;
+            // Decide on the run as it is NOW that the lease is held, never on the listing. The listing is a
+            // snapshot taken before this loop reached the row, and a live run moves on in between: the turn
+            // that was executing it settles and releases its lease, a fast step result resumes it, it may
+            // even complete. Trusting the listed `running` reads that as a crashed turn — and the code below
+            // counts a recovery attempt (or, past `maxRecoveryAttempts`, writes `dead` over a live or
+            // finished run) and re-enqueues it for a second execution. Only a run that still reads
+            // `running` under the lease is an orphan; release the lease taken just to look at anything else.
+            const run = await this.store.getRun(listed.id);
+            if (run?.status !== 'running') {
+                await this.store.releaseRunLock(listed.id, this.instanceId);
+                continue;
+            }
             // Count the attempt / dead-letter a poison pill past maxRecoveryAttempts.
             const settled = await this.countRecovery(run);
+            // `null`: the run left `running` between the re-read above and the dead-letter write (a
+            // lease-free `cancel()`), so it is not ours to touch — countRecovery already released the lease.
+            if (settled === null)
+                continue;
             if (settled) {
                 results.push(settled);
                 continue;
@@ -1306,7 +1329,8 @@ export class WorkflowEngine {
     /**
      * Per-recovery bookkeeping (called once the lease is held): count the attempt, or — past
      * `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
-     * result to skip the resume, or `undefined` to proceed.
+     * result to skip the resume, `null` to skip it silently (the run is no longer `running`), or
+     * `undefined` to proceed.
      */
     async countRecovery(run) {
         const attempts = (run.recoveryAttempts ?? 0) + 1;
@@ -1315,8 +1339,17 @@ export class WorkflowEngine {
                 message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) — moved to dead-letter`,
                 code: 'max_recovery_attempts',
             };
-            await this.store.updateRun(run.id, { status: 'dead', error, updatedAt: new Date() });
+            // Conditional on still-`running`, like the `pending` flip in recoverIncomplete: the lease does
+            // not fence `cancel()`, which writes `cancelled` without it, so a cancel landing after the
+            // caller's re-read must not be overwritten with `dead`.
+            const applied = await this.store.updateRunIf(run.id, ['running'], {
+                status: 'dead',
+                error,
+                updatedAt: new Date(),
+            });
             await this.store.releaseRunLock(run.id, this.instanceId);
+            if (!applied)
+                return null;
             this.emit({
                 type: 'run.failed',
                 runId: run.id,
@@ -2429,9 +2462,205 @@ export class WorkflowEngine {
         }
         // continue: persist any local steps the replay ran, dispatch the blocking ops, then suspend. When
         // those resolve (a result lands, a timer fires) `resume` brings us back for the next turn.
-        const wakeAt = await this.applyCommands(run, decision.commands);
+        const wakeAt = await this.applyCommands(run, decision.commands, true);
         return this.settleRun(run, { kind: 'suspended', wakeAt });
     }
+    /**
+     * A polyglot workflow's re-emitted `call` whose checkpoint already exists: decide whether the step is
+     * still out for delivery, or whether its dispatch was LOST and has to be re-driven. Returns the
+     * deadline to park the run on (so the next decision lands exactly when it's due), or `undefined` when
+     * there is nothing to wait for.
+     *
+     * WHY THIS EXISTS. A worker process killed mid-step (an OOM kill, exit 137 — a pod's memory ceiling,
+     * not a bug in anyone's code) takes its in-flight job with it, and the transport cannot always put it
+     * back: the `bullmq` transport's terminal-failure bridge (`worker.on('failed')`) only exists inside a
+     * JS consumer, so a first-class Python fleet (see `docs/python.mdx`) has none; the `queue` transport's
+     * reclaim sweep gives up past `maxStalledCount`, and never runs at all on an adapter without
+     * `recoverStalledJobs`. NOTHING then settles the checkpoint. It stays `pending` — "out for delivery" —
+     * and the `applyCommands` idempotency guard, which exists so a `ctx.gather_calls` fan-out doesn't
+     * double-dispatch its still-in-flight siblings on every partial resume, skips it on every later turn.
+     * The run wakes on the reconcile sweep, dispatches a turn, has the call re-emitted, skips it again,
+     * and sleeps. Forever — `pending`, `attempts = 1`, `wakeAt` NULL.
+     *
+     * THE POLICY (mirrors {@link callRemote}'s `ctx.step` self-heal, which this path silently did not
+     * share — so `remoteRedispatchMs` simply did not apply to a fan-out):
+     *  - a SETTLED checkpoint always wins: a step that completed just before the crash is never re-run;
+     *  - `remoteRedispatchMs` unset (the default) keeps the by-design "re-suspend, never re-dispatch";
+     *  - otherwise the pending checkpoint carries a LEASE in `wakeAt` — a result or a step heartbeat
+     *    ({@link rearmStepLease}) renews it, so a live worker still holding the step is never
+     *    double-dispatched — and only a LAPSED lease re-dispatches, bounded by `remoteRedispatchMax`;
+     *  - past the bound the step is failed `remote_step_lost` and the failure enters the run's history
+     *    (see {@link remoteHistory}), so the workflow's own error path surfaces it on the next turn
+     *    instead of the engine looping. The run is parked to wake immediately for exactly that turn.
+     */
+    async redriveLostCall(run, cp) {
+        // Settled (or a non-remote seq): the checkpoint is the truth, nothing to re-drive.
+        if (cp.kind !== 'remote' || cp.status !== 'pending')
+            return undefined;
+        if (this.remoteRedispatchMs == null)
+            return undefined;
+        // First sight of this pending step under the self-heal: stamp its lease (clock-space, persisted,
+        // stable across replays and crashes) and park the run on it.
+        if (cp.wakeAt == null) {
+            const lease = this.clock() + this.remoteRedispatchMs;
+            await this.store.saveCheckpoint({ ...cp, wakeAt: lease });
+            return this.stepLeaseWakeAt(lease);
+        }
+        if (this.clock() < cp.wakeAt)
+            return this.stepLeaseWakeAt(cp.wakeAt);
+        // The lease lapsed with no result and no heartbeat: the dispatch is presumed lost.
+        if (cp.attempts >= this.remoteRedispatchMax) {
+            const error = {
+                message: `remote step "${cp.name}" lost — no result after ${cp.attempts} dispatch(es)`,
+                code: 'remote_step_lost',
+                retryable: false,
+            };
+            const now = new Date();
+            await this.store.saveCheckpoint({
+                ...cp,
+                status: 'failed',
+                error,
+                wakeAt: undefined,
+                finishedAt: now,
+                events: [...(cp.events ?? []), lostStepEvent(this.clock(), cp.attempts, 'lost')],
+            });
+            this.emit({
+                type: 'step.failed',
+                runId: run.id,
+                seq: cp.seq,
+                name: cp.name,
+                kind: 'remote',
+                error,
+            });
+            // Wake now: the failure is in history, so the very next turn's replay raises it inside the
+            // gather and the run settles on the workflow's own error path.
+            return this.clock();
+        }
+        const attempts = cp.attempts + 1;
+        const lease = this.clock() + this.remoteRedispatchMs;
+        const at = new Date();
+        await this.store.saveCheckpoint({
+            ...cp,
+            attempts,
+            wakeAt: lease,
+            // The re-dispatch restarts the step's wall-clock window; the old one measured a worker that died.
+            enqueuedAt: at,
+            startedAt: at,
+            finishedAt: at,
+            // Observability: the step's own trail says WHY it ran again — a lost worker, not a failure retry.
+            events: [...(cp.events ?? []), lostStepEvent(this.clock(), attempts, 'redispatched')],
+        });
+        await this.dispatchRemoteTask({
+            runId: run.id,
+            seq: cp.seq,
+            name: cp.name,
+            stepId: cp.stepId ?? stepId(run.id, cp.seq),
+            group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cp.name), run.namespace),
+            input: cp.input,
+            attempt: attempts,
+        });
+        this.emit({
+            type: 'step.started',
+            runId: run.id,
+            seq: cp.seq,
+            name: cp.name,
+            kind: 'remote',
+            redispatched: true,
+        });
+        return this.stepLeaseWakeAt(lease);
+    }
+    /**
+     * The run `wakeAt` a step lease should park on: the lease itself, but never LATER than the
+     * {@link reconcileMs} safety net would have woken the run anyway. A long lease (a fan-out step that
+     * legitimately runs for half an hour) must not stretch the orphan sweep's cadence for this run — an
+     * early wake just re-parks on the un-lapsed lease, which is cheap and convergent.
+     */
+    stepLeaseWakeAt(lease) {
+        const reconcile = this.reconcileMs;
+        return reconcile == null ? lease : Math.min(lease, this.clock() + reconcile);
+    }
+    /** Earliest clock time a step's lease is worth LOOKING at again (`stepId` → clock ms) — the read
+     *  throttle for {@link rearmStepLease}, the sibling of {@link heartbeatPersists}. */
+    stepLeaseRearms = new Map();
+    /**
+     * A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
+     * that is still working on it is never presumed dead and double-dispatched. The durable counterpart
+     * of the in-memory `heartbeatResets` rearm (which only exists on the instance that dispatched a
+     * `timeoutMs` step). No-op unless the lost-dispatch self-heal is on — without `remoteRedispatchMs`
+     * nothing ever re-dispatches, so there is no lease to defend.
+     *
+     * THROTTLED, for the same reason {@link persistHeartbeat} is: beats arrive every few seconds from a
+     * hot batch loop, and defending a lease must not turn into a per-beat SELECT + UPDATE storm. The
+     * throttle is the lease itself rather than a fixed interval — a renewal is only WORTH a write once
+     * the window is half spent, because it buys at most one window either way. So:
+     *  - a beat that finds more than half the window still on the clock writes nothing, and records the
+     *    half-spent mark so subsequent beats skip even the `getCheckpoint` READ until then;
+     *  - a beat at or past the half-spent mark renews, leaving a full half-window of headroom — the
+     *    lease can never lapse under a worker whose beats are anywhere near their normal cadence.
+     *
+     * That bounds the cost at ONE read + ONE write per half `remoteRedispatchMs` per in-flight step
+     * (with a 30-minute window: two writes an hour), independent of beat frequency. The in-memory mark
+     * is a pure optimisation — cold on a fresh instance, or on the one that didn't dispatch the step —
+     * and the durable half-spent check re-derives the same answer from the checkpoint, so nothing
+     * depends on it surviving. Cleared wholesale past 1024 entries: a rare extra read, never a leak.
+     *
+     * NEVER REJECTS. This runs on the transport's SERIAL heartbeat handler, so a store hiccup thrown out
+     * of here would take the whole fleet's beat loop with it — and every lease would then lapse at once,
+     * a far worse failure than the write storm the throttle removes. So the entire rearm is best-effort,
+     * exactly like {@link persistHeartbeat}: a missed renewal costs at most one re-drive of a step that
+     * is idempotent by contract, and because the in-memory mark is advanced only AFTER a successful
+     * write, the very next beat retries the renewal rather than waiting out a window.
+     */
+    async rearmStepLease(runId, seq) {
+        if (this.remoteRedispatchMs == null)
+            return;
+        try {
+            await this.renewStepLease(runId, seq);
+        }
+        catch {
+            /* best-effort: a failed renewal must not break the transport's serial heartbeat loop */
+        }
+    }
+    /** {@link rearmStepLease}'s body — every durable read/write of the lease renewal, so the caller's one
+     *  guard covers all of them (the read, the checkpoint write, and the run's wake carry-forward). */
+    async renewStepLease(runId, seq) {
+        if (this.remoteRedispatchMs == null)
+            return;
+        const id = stepId(runId, seq);
+        const now = this.clock();
+        const lookAgainAt = this.stepLeaseRearms.get(id);
+        if (lookAgainAt !== undefined && now < lookAgainAt)
+            return;
+        const cp = await this.store.getCheckpoint(runId, seq);
+        if (cp?.kind !== 'remote' || cp.status !== 'pending') {
+            // Settled (or gone): stop tracking it, so a long-lived worker's stale beats don't keep the entry.
+            this.stepLeaseRearms.delete(id);
+            return;
+        }
+        const halfWindow = Math.ceil(this.remoteRedispatchMs / 2);
+        // More than half the window left: nothing to buy, and nothing to look at until it is half spent.
+        if (cp.wakeAt != null && cp.wakeAt > now + halfWindow) {
+            this.stepLeaseRearms.set(id, cp.wakeAt - halfWindow);
+            return;
+        }
+        const renewed = now + this.remoteRedispatchMs;
+        const previous = cp.wakeAt;
+        await this.store.saveCheckpoint({ ...cp, wakeAt: renewed });
+        if (this.stepLeaseRearms.size > 1024)
+            this.stepLeaseRearms.clear();
+        this.stepLeaseRearms.set(id, renewed - halfWindow);
+        // If the run is parked EXACTLY on this step's old lease, carry its wake forward too — otherwise a
+        // beating worker costs one wasted turn per renewal window. Only on an exact match: any other
+        // `wakeAt` belongs to some other timer (a sleep, a sibling's shorter lease) and moving it would
+        // delay that instead. Failing HERE (after the checkpoint write landed) is harmless: the run wakes
+        // on the old lease, finds it un-lapsed, and re-parks on the renewed one.
+        if (previous == null)
+            return;
+        const run = await this.store.getRun(runId);
+        if (run?.status === 'suspended' && run.wakeAt === previous) {
+            await this.store.updateRun(runId, { wakeAt: renewed, updatedAt: new Date() });
+        }
+    }
     /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
     async remoteHistory(runId) {
         const checkpoints = await this.store.listCheckpoints(runId);
@@ -2475,8 +2704,13 @@ export class WorkflowEngine {
         return events.sort((a, b) => a.seq - b.seq);
     }
     /** Apply a turn's commands: persist recorded local steps, dispatch remote calls, schedule timers.
-     *  Returns the earliest timer deadline to suspend on (or undefined — suspended on a result). */
-    async applyCommands(run, commands) {
+     *  Returns the earliest timer deadline to suspend on (or undefined — suspended on a result).
+     *
+     *  `redriveLost` is set only by the `continue` path: a turn that PARKS on its blocking ops is the one
+     *  that may find a re-emitted `call` whose dispatch was lost and needs re-driving (see
+     *  {@link redriveLostCall}). A terminal turn (completed/failed/cancelled) applies its commands only to
+     *  record what already ran, and must never dispatch anything new. */
+    async applyCommands(run, commands, redriveLost = false) {
         let wakeAt;
         for (const cmd of commands) {
             const at = new Date();
@@ -2519,13 +2753,29 @@ export class WorkflowEngine {
                 // already exists — pending OR terminal — skip the save + dispatch entirely; its result lands
                 // independently via the remote-result path (keyed by seq, so concurrent calls never clobber).
                 // Mirrors the `startChild` `getRun(childId)` guard below.
-                if (await this.store.getCheckpoint(run.id, cmd.seq))
+                //
+                // What that guard must NOT do is treat "a checkpoint exists" as "somebody is still going to
+                // deliver it". A dispatched job can be LOST — the worker was killed mid-step, so the job died
+                // with the process and no result will ever come — and the checkpoint still reads `pending`.
+                // Skipping such a command on every later turn is how a fan-out orphans its run (see
+                // {@link redriveLostCall}, which owns that decision; a settled checkpoint always wins).
+                const existing = await this.store.getCheckpoint(run.id, cmd.seq);
+                if (existing) {
+                    const leaseAt = redriveLost ? await this.redriveLostCall(run, existing) : undefined;
+                    if (leaseAt != null)
+                        wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
                     continue;
+                }
                 // Route this cross-SDK-worker `call` by the SAME name-based token a native `ctx.step`
                 // dispatches with (and a worker subscribes to per handler name), so a decision-driven remote
                 // step and an in-process one land on the identical queue. The command's own `group` is not a
                 // routing input; the run's namespace is, so a tenant's fan-out stays on the tenant's pool.
                 const callToken = tenantGroup(sanitizeQueueToken(cmd.name), run.namespace);
+                // The dispatched step's LEASE: when the lost-dispatch self-heal is on (`remoteRedispatchMs`),
+                // stamp the deadline by which a result — or a step heartbeat renewing it — must have landed,
+                // and suspend the run ON it, so the re-drive lands when it's due instead of waiting for the
+                // reconcile sweep. Unset (the default) leaves `wakeAt` NULL exactly as before.
+                const lease = this.remoteRedispatchMs == null ? undefined : this.clock() + this.remoteRedispatchMs;
                 await this.store.saveCheckpoint(stepCheckpoint({
                     runId: run.id,
                     seq: cmd.seq,
@@ -2534,6 +2784,7 @@ export class WorkflowEngine {
                     status: 'pending',
                     input: cmd.input,
                     attempts: 1,
+                    wakeAt: lease,
                     workerGroup: callToken,
                     enqueuedAt: at,
                     startedAt: at,
@@ -2561,6 +2812,10 @@ export class WorkflowEngine {
                     name: cmd.name,
                     kind: 'remote',
                 });
+                if (lease != null) {
+                    const leaseWake = this.stepLeaseWakeAt(lease);
+                    wakeAt = wakeAt == null ? leaseWake : Math.min(wakeAt, leaseWake);
+                }
             }
             else if (cmd.kind === 'sleep') {
                 const deadline = this.clock() + cmd.ms;
@@ -2721,19 +2976,26 @@ export class WorkflowEngine {
         for (const cp of snapshot)
             replay.set(cp.seq, cp);
         const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow);
+        // Every settle below is `return await`ed, never a bare `return this.settleRun(...)`: inside
+        // `try … finally` a bare return runs the `finally` — which RELEASES THE LEASE — as soon as the
+        // return expression is evaluated, while the settle's own write is still in flight. A first turn
+        // (the one that flipped `pending` -> `running`) then sits unlocked but still reading `running`,
+        // which is exactly the shape `listOrphanedRuns` returns for a crashed turn: the recovery sweep
+        // counted an attempt (or dead-lettered it), flipped it to `pending` and re-enqueued a second
+        // execution. The lease must outlive the state it protects.
         try {
             // Establish the ambient ctx for the duration of this body turn so context-aware statics
             // (`BaseWorkflow.start`/`dispatch`) reachable from `fn` route through this run's `ctx.child`/
             // `ctx.startChild`. Re-set on every replay turn (each is its own async scope) — correct.
             const output = await workflowAls.run(ctx, () => fn(ctx, run.input));
-            return this.settleRun(run, { kind: 'completed', output });
+            return await this.settleRun(run, { kind: 'completed', output });
         }
         catch (err) {
             if (err instanceof ContinueAsNew) {
                 // Hand off to a fresh execution with a clean history: persist the next run (`<id>~N`), then
                 // complete this one. Zombie fence first — a fenced turn must not fork a continuation either.
                 if (this.leaseLostRuns.has(run.id))
-                    return this.echoCurrentStatus(run.id);
+                    return await this.echoCurrentStatus(run.id);
                 const nextId = nextContinuationId(run.id);
                 // Persist the continuation BEFORE the parent's terminal write — the durable ordering that
                 // makes the chain crash-safe. The old order (complete parent, then create the continuation
@@ -2770,7 +3032,7 @@ export class WorkflowEngine {
                 // No live worker can run the next dispatch (capability/protocol) — park `blocked` with the
                 // structured reason + diagnostics instead of suspending on a timer that would re-drive into the
                 // same void. The blocked-recovery poll re-drives it when a capable+compatible worker appears.
-                return this.parkBlocked(run, err.plan);
+                return await this.parkBlocked(run, err.plan);
             }
             if (err instanceof WorkflowSuspended) {
                 // A compensating cancel resumed this run to reach here: the replay re-registered the saga,
@@ -2840,7 +3102,7 @@ export class WorkflowEngine {
                     continue;
                 await this.runCompensation(run, comp, i);
             }
-            return this.settleRun(run, { kind: 'failed', error });
+            return await this.settleRun(run, { kind: 'failed', error });
         }
         finally {
             // Release the recovery lease once the run reaches a terminal/suspended state, so the
@@ -3299,7 +3561,16 @@ export class WorkflowEngine {
     async callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace) {
         // Read the prefix from the per-execution snapshot (avoids the O(N²) replay SELECTs); a seq absent
         // from the snapshot — not yet dispatched, or written after the snapshot — falls back to the store.
-        const existing = replay?.get(seq) ?? (await this.store.getCheckpoint(runId, seq));
+        let existing = replay?.get(seq) ?? (await this.store.getCheckpoint(runId, seq));
+        // A `pending` step in the SNAPSHOT may already have its result: the snapshot is taken when this
+        // execution starts, and the result can land any time after. Every branch below that sees
+        // `pending` either suspends on it or WRITES the checkpoint back (`{ ...existing, wakeAt }` stamping
+        // the lost-dispatch lease, or a re-dispatch) — written from a stale row, that overwrites the landed
+        // `completed` result with `pending` and parks the run on a lease an hour out. So a pending step is
+        // re-read from the store before anything acts on it (one read, only at a suspension point).
+        if (existing?.status === 'pending' && replay?.has(seq)) {
+            existing = (await this.store.getCheckpoint(runId, seq)) ?? existing;
+        }
         if (existing && existing.name !== step.name) {
             throw new NonDeterminismError(runId, seq, step.name, existing.name);
         }
@@ -3350,6 +3621,12 @@ export class WorkflowEngine {
                 enqueuedAt: reEnqueuedAt,
                 startedAt: reEnqueuedAt,
                 finishedAt: reEnqueuedAt,
+                // Same observable marker the polyglot `call` path stamps (see redriveLostCall): the step's own
+                // trail distinguishes a lost-dispatch re-drive from a failure retry.
+                events: [
+                    ...(existing.events ?? []),
+                    lostStepEvent(this.clock(), reAttempt, 'redispatched'),
+                ],
             });
             await this.dispatchRemoteTask({
                 runId,
@@ -3362,6 +3639,14 @@ export class WorkflowEngine {
                 attempt: reAttempt,
                 transport,
             });
+            this.emit({
+                type: 'step.started',
+                runId,
+                seq,
+                name: step.name,
+                kind: 'remote',
+                redispatched: true,
+            });
             throw new WorkflowSuspended(nextDeadline, [seq]);
         }
         // Durable retry: a failed attempt re-dispatches up to `retries`, spacing attempts by `backoff` —