@adonis-agora/durable 0.47.0 → 0.49.0

package/dist/src/engine.js

@@ -1,17 +1,19 @@
 import { InMemoryAdmissionBackend } from './admission.js';
 import { backoffDelay, MAX_BACKOFF_MS } from './backoff.js';
 import { instantCheckpoint, lostStepEvent, stepCheckpoint } from './checkpoints.js';
-import { isFailedCompletion } from './completion.js';
+import { completionError, failedCompletion, isFailedCompletion, } from './completion.js';
 import { assertConcurrency, concurrencyTag, countRuns, resolveConcurrency, } from './concurrency.js';
 import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispatch-routing.js';
 import { parseDuration } from './duration.js';
 import { Entities } from './entities.js';
-import { ContinueAsNew, NonDeterminismError, RemoteStepTimeout, WorkflowSuspended, } from './errors.js';
+import { ContinueAsNew, GatherError, NonDeterminismError, RemoteStepTimeout, WorkflowSuspended, } from './errors.js';
 import { EventAccumulators } from './event-accumulators.js';
 import { eventMatches, eventMatchOf, eventPrefix } from './events.js';
-import { FOLDED_PREFIX, fanOfWaiter, foldedChildIds } from './fan-join.js';
+import { childIdsOf, FOLDED_PREFIX, fanOfWaiter, parseStepFanGroup, stepFanGroup, } from './fan-join.js';
+import { aggregateAnnouncements } from './handshake/announced.js';
 import { isNonEmptyRunScope, TERMINAL_RUN_STATUSES } from './interfaces.js';
 import { asHeartbeat } from './observability-scope.js';
+import { assertRoutes, codeAndDetails, HANDOFF_SEQ_BASE, handoffHandlerId, handoffName, liveHandoff, onErrorRunId, retiredHandoffName, routeFor, } from './on-error.js';
 import { breakpointToken, stepId } from './protocol.js';
 import { RemoteWorkflowExecutor } from './remote-workflow-executor.js';
 import { scanRunValueFacets } from './run-value-facets.js';
@@ -58,6 +60,8 @@ const SETTLE_ELIGIBLE_STATUSES = [
     'blocked',
     'failed',
 ];
+/** How long a read of the onError routes a fleet announces is reused. */
+const ANNOUNCED_ROUTES_TTL_MS = 5_000;
 const versionKey = (name, version) => `${name}@${version}`;
 /**
  * First checkpoint seq reserved for saga compensations: compensation `idx` (registration order)
@@ -66,6 +70,18 @@ const versionKey = (name, version) => `${name}@${version}`;
  * un-checkpointed `-1` undo dispatches of older engine versions.
  */
 const COMPENSATION_SEQ_BASE = -2;
+/** How many checkpoint writes / dispatches a step fan sends at once. */
+const STEP_FAN_CHUNK = 200;
+/** The default {@link WorkflowEngineDeps.timerBatchSize}. */
+const DEFAULT_TIMER_BATCH = 500;
+/** Fisher–Yates, in place. */
+function shuffle(items) {
+    for (let i = items.length - 1; i > 0; i -= 1) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [items[i], items[j]] = [items[j], items[i]];
+    }
+    return items;
+}
 /**
  * The routing token a dispatched step lands on: the step's own (broker-safe) name, suffixed by the
  * tenant the RUN belongs to. `runNamespace` — not the executing engine's namespace — is the tenant
@@ -129,6 +145,7 @@ export class WorkflowEngine {
     leaseMs;
     maxRecoveryAttempts;
     reconcileMs;
+    timerBatchSize;
     remoteRedispatchMs;
     remoteRedispatchMax;
     webhookUrl;
@@ -198,6 +215,8 @@ export class WorkflowEngine {
     updateValidators = new Map();
     /** Runs being cancelled WITH saga compensation — see `cancel({ compensate: true })`. */
     cancelRequested = new Set();
+    /** {@link announcedRoutes} reads, by routing token + workflow + version. */
+    announcedRoutesCache = new Map();
     /** The merged schedule set the worker loop ticks with (see {@link adoptSchedules}). */
     #allSchedules = [];
     /** Runtime pause overrides by schedule key (console-issued; win over the config's `paused`). */
@@ -278,6 +297,12 @@ export class WorkflowEngine {
         this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
         // Default 5min; an explicit 0 disables the fallback (opt back into wake-forever-on-lost-event).
         this.reconcileMs = deps.reconcileMs === undefined ? 300_000 : deps.reconcileMs || undefined;
+        // A non-finite or sub-1 value falls back to the default: a NaN limit would stall timer recovery.
+        const batch = deps.timerBatchSize;
+        this.timerBatchSize =
+            batch !== undefined && Number.isFinite(batch) && batch >= 1
+                ? Math.floor(batch)
+                : DEFAULT_TIMER_BATCH;
         // Opt-in: unset (or 0) leaves the by-design "re-suspend a lost dispatch, never re-dispatch" behavior.
         this.remoteRedispatchMs = deps.remoteRedispatchMs || undefined;
         this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
@@ -545,6 +570,7 @@ export class WorkflowEngine {
             name,
             version,
             fn,
+            onError: this.checkedRoutes(name, opts?.onError),
             tags: opts?.tags,
             origin: opts?.origin,
             singleton: opts?.singleton,
@@ -575,6 +601,7 @@ export class WorkflowEngine {
         const registered = {
             name,
             version,
+            onError: this.checkedRoutes(name, opts.onError),
             // A remote workflow has no in-process body; execute() branches on `remote` before this is read.
             fn: () => {
                 throw new Error(`workflow ${name} is remote — it has no in-process body`);
@@ -613,6 +640,9 @@ export class WorkflowEngine {
             group: name,
             executor: new RemoteWorkflowExecutor(this.primaryTransport, name),
         });
+        const registered = this.workflows.get(versionKey(name, '1'));
+        if (registered)
+            registered.convention = true;
         return true;
     }
     /**
@@ -1350,6 +1380,31 @@ export class WorkflowEngine {
             }
         })());
     }
+    /**
+     * {@link recheckWaitSeqs} for the step fans a suspended turn waits on: re-drive the run once a fan
+     * has no step left `pending`. The last result of a fan may have opened the join barrier while this
+     * turn held the lease, and its resume no-oped against it. A fan with a step still `pending` is left
+     * to that step's result.
+     */
+    recheckStepFans(runId, fans) {
+        this.trackEffect((async () => {
+            await new Promise((r) => setTimeout(r, 0));
+            for (const fan of fans) {
+                const pending = await this.countPendingSteps(runId, fan.from, fan.to).catch(() => 1);
+                if (pending === 0) {
+                    await this.resumeImplicit(runId).catch(() => undefined);
+                    return;
+                }
+            }
+        })());
+    }
+    /** {@link StateStore.countCheckpoints} of `pending` steps, or a JS count on a store without it. */
+    async countPendingSteps(runId, from, to) {
+        if (this.store.countCheckpoints) {
+            return this.store.countCheckpoints(runId, from, to, ['pending']);
+        }
+        return (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= from && c.seq <= to && c.status === 'pending').length;
+    }
     /** Track an in-flight execution so {@link drain} can wait for it. */
     track(runId, p) {
         this.inflight.set(p, runId);
@@ -1625,8 +1680,10 @@ export class WorkflowEngine {
      * not due re-suspends cheaply without running new work.
      */
     async resumeDueTimers(nowMs = this.clock()) {
-        // Capped per tick (a backlog drains over several polls) — a store may ignore the cap.
-        const due = await asHeartbeat(() => this.store.listDueTimers(nowMs, this.namespace, 200));
+        // Capped per tick (a backlog drains over several polls) — a store may ignore the cap. Shuffled:
+        // every instance loads the same most-overdue batch, and walking it in the same order would have
+        // them all contend for the same leases one row at a time.
+        const due = shuffle(await asHeartbeat(() => this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize)));
         const blocked = await this.dueBlockedRuns(nowMs);
         return this.resumeLeased([...due, ...blocked], nowMs);
     }
@@ -1689,6 +1746,24 @@ export class WorkflowEngine {
         if (run.status === 'failed' || run.status === 'dead') {
             const isFailureCompletion = (v) => typeof v === 'object' && v !== null && v.ok === false;
             for (const cp of await this.store.listCheckpoints(runId)) {
+                // A retry runs the body again, so a failure it handed to an onError handler is retired: kept in
+                // the timeline under another name, never followed again, and the handler is not retried. Its
+                // `onError:<code>` tag goes too: the tag says how the run ended, and that is being redone.
+                const handlerId = handoffHandlerId(runId, cp);
+                if (handlerId !== undefined) {
+                    await this.store.takeSignalWaiter(`child:${handlerId}`);
+                    await this.store.saveCheckpoint({
+                        ...cp,
+                        name: retiredHandoffName(handlerId),
+                        status: 'completed',
+                    });
+                    const tags = run.tags?.filter((t) => !t.startsWith('onError:'));
+                    if (tags && tags.length !== run.tags?.length) {
+                        await this.store.updateRun(runId, { tags, updatedAt: new Date() });
+                        run.tags = tags;
+                    }
+                    continue;
+                }
                 if (cp.status === 'failed') {
                     await this.store.saveCheckpoint({
                         ...cp,
@@ -2208,7 +2283,7 @@ export class WorkflowEngine {
             ? buffered.payload
             : settled.status === 'completed'
                 ? { ok: true, value: settled.output }
-                : { ok: false, error: settled.error?.message ?? settled.status };
+                : failedCompletion(settled.error ?? { message: settled.status });
         await this.signal(token, completion);
     }
     /**
@@ -2597,6 +2672,169 @@ export class WorkflowEngine {
      * {@link runExecution}'s settle/suspend; the lease is held by {@link execute}. The result that lands
      * us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
      */
+    /**
+     * The registered onError routes whose handler workflow nothing can run: not registered on this
+     * engine, and no live worker group serves it. Meant for a boot-time warning: a typo in a handler's
+     * name would otherwise surface only when the first run fails.
+     */
+    async unservedOnErrorRoutes() {
+        const unserved = [];
+        let groups;
+        for (const registered of this.workflows.values()) {
+            for (const route of registered.onError ?? []) {
+                if (this.latest.has(route.workflow))
+                    continue;
+                groups ??= await this.pool.listWorkerGroups().catch(() => []);
+                if (groups.some((g) => g === route.workflow || g.startsWith(`${route.workflow}@`)))
+                    continue;
+                unserved.push({ workflow: registered.name, route });
+            }
+        }
+        return unserved;
+    }
+    /**
+     * The onError routes the live workers on `token` announce for a workflow the engine serves by
+     * convention: the one set they agree on, or none when they state none or cannot be read. Workers
+     * that announce different sets (a rollout in progress) route nothing, and the run's failure is
+     * reported as a `run.unrouted` event. Announcers of another version of the workflow are ignored. The
+     * read is cached for {@link ANNOUNCED_ROUTES_TTL_MS}, so a burst of failures costs one read.
+     */
+    async announcedRoutes(run, token, error) {
+        const key = `${token}\n${run.workflow}\n${run.workflowVersion}`;
+        const now = this.clock();
+        let cached = this.announcedRoutesCache.get(key);
+        if (!cached || now - cached.at > ANNOUNCED_ROUTES_TTL_MS) {
+            const sets = new Map();
+            for (const announced of aggregateAnnouncements(await this.liveDescriptorsFor(token))) {
+                if (announced.name !== run.workflow)
+                    continue;
+                if (announced.version !== undefined && announced.version !== run.workflowVersion)
+                    continue;
+                for (const routes of announced.onError)
+                    sets.set(JSON.stringify(routes), routes);
+            }
+            cached = { at: now, sets: [...sets.values()] };
+            this.announcedRoutesCache.set(key, cached);
+        }
+        if (cached.sets.length > 1) {
+            this.emit({
+                type: 'run.unrouted',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+                error,
+                reason: `live workers announce ${cached.sets.length} different onError route sets`,
+            });
+            return undefined;
+        }
+        return cached.sets[0];
+    }
+    /** Validate a workflow's {@link OnErrorRoute}s at registration. */
+    checkedRoutes(name, routes) {
+        if (!routes?.length)
+            return undefined;
+        assertRoutes(name, routes);
+        return routes;
+    }
+    /**
+     * Route a failed run to its {@link OnErrorRoute} handler. Starts the handler run and waits for it the
+     * way a parent waits for a `ctx.child`: a `child:<handlerId>` waiter plus a `running` placeholder
+     * checkpoint carrying the handler's name and input, so the handler's terminal state wakes the run
+     * (buffered and reconciled like any child). From then on every execution of the run only follows the
+     * handler ({@link followHandoff}); the body never runs again. Undefined when no route matches: the
+     * run fails as usual.
+     *
+     * The placeholder sits past every seq the body used and is left out of {@link remoteHistory}.
+     */
+    async routeFailure(run, error) {
+        const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
+        const routes = registered?.convention
+            ? await this.announcedRoutes(run, registered.remote?.group ?? run.workflow, error)
+            : registered?.onError;
+        const route = routeFor(routes, error);
+        if (!route)
+            return undefined;
+        const checkpoints = await this.handoffCheckpoints(run.id);
+        const live = liveHandoff(run.id, checkpoints);
+        if (live)
+            return this.followHandoff(run, live);
+        // One handoff per attempt: a retry retires the previous one, so the next sits past it.
+        const attempt = checkpoints.length;
+        const seq = HANDOFF_SEQ_BASE + attempt;
+        const handlerId = onErrorRunId(run.id, attempt);
+        const handler = {
+            workflow: route.workflow,
+            input: { runId: run.id, workflow: run.workflow, input: run.input, error },
+        };
+        await this.store.putSignalWaiter({ token: `child:${handlerId}`, runId: run.id, seq });
+        await this.store.saveCheckpoint({
+            ...instantCheckpoint({
+                runId: run.id,
+                seq,
+                name: handoffName(handlerId),
+                kind: 'signal',
+                status: 'running',
+            }),
+            input: handler,
+        });
+        // Tagged, so the runs whose failures were handed off are a filter away in the console.
+        const tag = `onError:${route.code ?? '*'}`;
+        if (!run.tags?.includes(tag)) {
+            const tags = [...(run.tags ?? []), tag];
+            await this.store.updateRun(run.id, { tags, updatedAt: new Date() });
+            run.tags = tags;
+        }
+        this.emit({
+            type: 'run.routed',
+            runId: run.id,
+            workflow: run.workflow,
+            namespace: run.namespace,
+            name: route.workflow,
+            error,
+        });
+        this.startChildDeferred(handler.workflow, handler.input, handlerId, {
+            namespace: run.namespace,
+        });
+        return { kind: 'suspended' };
+    }
+    /** A run's handoff checkpoints, live and retired: the only ones at {@link HANDOFF_SEQ_BASE} and up. */
+    async handoffCheckpoints(runId) {
+        const handoffs = await this.store.listCheckpointsByNamePrefix?.(runId, [
+            `signal:child:${runId}.onError.`,
+            `onError:retried:${runId}.onError.`,
+        ]);
+        return (handoffs ?? (await this.store.listCheckpoints(runId))).filter((cp) => cp.seq >= HANDOFF_SEQ_BASE);
+    }
+    /**
+     * Where a run that handed its failure off stands: the handler's outcome once it settled (from the
+     * wake's checkpoint, or off the handler run when the wake never arrived), else still waiting. Starts
+     * the handler if the turn that handed off crashed before its deferred start ran.
+     */
+    async followHandoff(run, handoff) {
+        if (handoff.status === 'completed') {
+            const outcome = handoff.output;
+            return outcome.ok
+                ? { kind: 'completed', output: outcome.value }
+                : { kind: 'failed', error: completionError(outcome) };
+        }
+        const handlerId = handoffHandlerId(run.id, handoff);
+        const handlerRun = await this.store.getRun(handlerId);
+        if (handlerRun?.status === 'completed')
+            return { kind: 'completed', output: handlerRun.output };
+        if (handlerRun && TERMINAL_RUN_STATUSES.includes(handlerRun.status)) {
+            return {
+                kind: 'failed',
+                error: handlerRun.error ?? { message: `onError handler ${handlerId} ${handlerRun.status}` },
+            };
+        }
+        if (!handlerRun) {
+            const handler = handoff.input;
+            this.startChildDeferred(handler.workflow, handler.input, handlerId, {
+                namespace: run.namespace,
+            });
+        }
+        return { kind: 'suspended' };
+    }
     /**
      * Apply a terminal/suspended transition once: persist the new run status, emit the matching
      * lifecycle event, and (on completion/failure) wake a waiting parent. Both the TS and remote
@@ -2639,6 +2877,10 @@ export class WorkflowEngine {
             return { runId: run.id, status: 'completed', output: outcome.output };
         }
         if (outcome.kind === 'failed') {
+            const handed = await this.routeFailure(run, outcome.error);
+            if (handed && handed.kind !== 'failed')
+                return this.settleRun(run, handed);
+            const error = handed?.error ?? outcome.error;
             // Same race as the `completed` branch above — guard identically. ADDITIONALLY exclude
             // `failed`: a stray resume of an already-failed run (a redelivered signal, the settle-recheck
             // re-drive) replays and deterministically re-fails — re-applying the settle would re-emit
@@ -2649,7 +2891,7 @@ export class WorkflowEngine {
             // status), so it is unaffected.
             const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES.filter((s) => s !== 'failed'), {
                 status: 'failed',
-                error: outcome.error,
+                error,
                 updatedAt,
             });
             if (!applied)
@@ -2659,10 +2901,10 @@ export class WorkflowEngine {
                 runId: run.id,
                 workflow: run.workflow,
                 namespace: run.namespace,
-                error: outcome.error,
+                error,
             });
-            this.notifyParent(run.id, { ok: false, error: outcome.error.message });
-            return { runId: run.id, status: 'failed', error: outcome.error };
+            this.notifyParent(run.id, failedCompletion(error));
+            return { runId: run.id, status: 'failed', error };
         }
         // This outcome was computed by a turn that started from a possibly-stale run snapshot. If the run
         // was cancelled WHILE that turn was still executing (e.g. `ctx.all`'s failFast cancelling a
@@ -2746,7 +2988,13 @@ export class WorkflowEngine {
         const routePlan = await this.planRoute(remote.group, registered.requires);
         if (routePlan?.status === 'blocked')
             return this.parkBlocked(run, routePlan);
-        const history = await this.remoteHistory(run.id);
+        const checkpoints = await this.store.listCheckpoints(run.id);
+        const handoff = liveHandoff(run.id, checkpoints);
+        if (handoff) {
+            // A run that handed its failure to an onError handler only follows the handler from here on.
+            return this.settleRun(run, await this.followHandoff(run, handoff));
+        }
+        const history = await this.remoteHistory(run.id, checkpoints);
         let decision;
         try {
             decision = await remote.executor.advance(run, history);
@@ -3007,8 +3255,8 @@ export class WorkflowEngine {
         }
     }
     /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
-    async remoteHistory(runId) {
-        const checkpoints = await this.store.listCheckpoints(runId);
+    async remoteHistory(runId, listed) {
+        const checkpoints = listed ?? (await this.store.listCheckpoints(runId));
         const kindOf = {
             remote: 'call',
             local: 'step',
@@ -3021,13 +3269,16 @@ export class WorkflowEngine {
                 // A child run resolves THROUGH the signal machinery (a `child:<id>` waiter notified on the
                 // child's terminal state), so its checkpoint is kind `signal` with a `signal:child:` name and
                 // a Completion payload. Surface it as a `child` event with the value/error unwrapped.
+                // onError handoffs sit past every op a body can reach; no replay is told about them.
+                if (cp.seq >= HANDOFF_SEQ_BASE)
+                    continue;
                 if (cp.kind === 'signal' && cp.name.startsWith('signal:child:')) {
                     const completion = cp.output;
                     events.push({
                         seq: cp.seq,
                         kind: 'child',
                         output: completion?.ok ? completion.value : undefined,
-                        error: completion && completion.ok === false ? { message: completion.error } : undefined,
+                        error: completion && completion.ok === false ? completionError(completion) : undefined,
                     });
                     continue;
                 }
@@ -3270,7 +3521,7 @@ export class WorkflowEngine {
                         child.status === 'dead') {
                         await deliverChild(child.status === 'completed'
                             ? { ok: true, value: child.output }
-                            : { ok: false, error: child.error?.message ?? child.status });
+                            : failedCompletion(child.error ?? { message: child.status }));
                     }
                 }
             }
@@ -3330,6 +3581,11 @@ export class WorkflowEngine {
         // counted an attempt (or dead-lettered it), flipped it to `pending` and re-enqueued a second
         // execution. The lease must outlive the state it protects.
         try {
+            // A run that handed its failure to an onError handler only follows the handler from here on.
+            const handoff = liveHandoff(run.id, snapshot);
+            if (handoff && !this.cancelRequested.has(run.id)) {
+                return await this.settleRun(run, await this.followHandoff(run, handoff));
+            }
             // Establish the ambient ctx for the duration of this body turn so context-aware statics
             // (`BaseWorkflow.start`/`dispatch`) reachable from `fn` route through this run's `ctx.child`/
             // `ctx.startChild`. Re-set on every replay turn (each is its own async scope) — correct.
@@ -3432,6 +3688,9 @@ export class WorkflowEngine {
                 if (settled.status === 'suspended' && err.waitSeqs?.length) {
                     this.recheckWaitSeqs(run.id, err.waitSeqs);
                 }
+                if (settled.status === 'suspended' && err.stepFans?.length) {
+                    this.recheckStepFans(run.id, err.stepFans);
+                }
                 if (settled.status === 'suspended')
                     suspendedOnFan = err.fan;
                 return settled;
@@ -3439,6 +3698,7 @@ export class WorkflowEngine {
             const error = {
                 message: err instanceof Error ? err.message : String(err),
                 stack: err instanceof Error ? err.stack : undefined,
+                ...codeAndDetails(err),
             };
             // Saga: undo completed steps in reverse, each retried up to `compensationRetries`. Outcomes
             // are emitted as `compensate:<step>` step events so a stranded undo is VISIBLE (not silently
@@ -3717,6 +3977,7 @@ export class WorkflowEngine {
             completeStep: (s) => this.completeStep(s),
             failStep: (s) => this.failStep(s),
             callRemote: (runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace),
+            callRemoteAll: (runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, runNamespace),
             // Defer so a fast child can't reentrantly resume a still-running parent. The child inherits the
             // PARENT RUN's namespace, not this engine's: an operator legitimately executes every tenant's
             // parents, and without this the child would fall to the operator's own pool.
@@ -3910,7 +4171,7 @@ export class WorkflowEngine {
         }));
         return blocked;
     }
-    async callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace) {
+    async callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace, parallelGroup) {
         // Read the prefix from the per-execution snapshot (avoids the O(N²) replay SELECTs); a seq absent
         // from the snapshot — not yet dispatched, or written after the snapshot — falls back to the store.
         let existing = replay?.get(seq) ?? (await this.store.getCheckpoint(runId, seq));
@@ -3953,7 +4214,7 @@ export class WorkflowEngine {
                 // The result may have settled the row since it was read: then nothing was written, and the
                 // step is re-read from the store.
                 if (!(await this.savePendingCheckpoint({ ...existing, wakeAt }))) {
-                    return this.callRemote(runId, seq, step, input, queue, transport, undefined, admission, runNamespace);
+                    return this.callRemote(runId, seq, step, input, queue, transport, undefined, admission, runNamespace, parallelGroup);
                 }
                 throw new WorkflowSuspended(wakeAt, [seq]);
             }
@@ -3985,7 +4246,7 @@ export class WorkflowEngine {
                 ],
             });
             if (!claimed) {
-                return this.callRemote(runId, seq, step, input, queue, transport, undefined, admission, runNamespace);
+                return this.callRemote(runId, seq, step, input, queue, transport, undefined, admission, runNamespace, parallelGroup);
             }
             await this.dispatchRemoteTask({
                 runId,
@@ -4081,6 +4342,7 @@ export class WorkflowEngine {
             // released by WHICHEVER instance receives the result — the in-memory stepQueue map only
             // covers the dispatching pod, and a slot released by no one starves the queue fleet-wide.
             queue: admittedQueue,
+            ...(parallelGroup !== undefined ? { parallelGroup } : {}),
             enqueuedAt,
             startedAt: enqueuedAt, // placeholders until the worker result lands
             finishedAt: enqueuedAt,
@@ -4099,6 +4361,150 @@ export class WorkflowEngine {
         this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
         throw new WorkflowSuspended(undefined, [seq]);
     }
+    /**
+     * `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
+     * reserves the whole position block up front, so replay re-derives the same seqs.
+     *
+     * Each turn reads the fan's checkpoints fresh, hands every step that already has one to
+     * {@link callRemote} (which owns the completed / pending / failed-retry rules), and sends the steps
+     * nobody has sent yet: all of them, or with `concurrency`, the next wave once the last one has
+     * drained. New steps are written `pending` BEFORE any is dispatched, so a turn that runs after a
+     * fast result finds them already sent instead of sending them again. The suspension carries the
+     * fan's block, not its seqs, so the join barrier ({@link stepFanStillRunning}) and
+     * {@link recheckStepFans} wake the parent once per wave, not once per result.
+     */
+    async callRemoteAll(runId, positions, step, inputs, opts, runNamespace) {
+        if (step.timeoutMs) {
+            throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
+        }
+        if (opts.concurrency !== undefined &&
+            (!Number.isInteger(opts.concurrency) || opts.concurrency < 1)) {
+            throw new Error(`ctx.all: concurrency must be a positive integer, got ${opts.concurrency}`);
+        }
+        if (this.pool.size === 0)
+            throw new Error('remote steps require a Transport');
+        const first = positions[0];
+        const last = positions[positions.length - 1];
+        const group = stepFanGroup(first, positions.length, opts.mode);
+        const bySeq = new Map();
+        for (const cp of await this.store.listCheckpoints(runId)) {
+            if (cp.seq >= first && cp.seq <= last)
+                bySeq.set(cp.seq, cp);
+        }
+        const outputs = new Array(inputs.length);
+        const failures = [];
+        const unsent = [];
+        let inFlight = 0;
+        let wakeAt;
+        for (let i = 0; i < inputs.length; i += 1) {
+            const seq = positions[i];
+            const cp = bySeq.get(seq);
+            if (!cp) {
+                unsent.push(i);
+                continue;
+            }
+            if (cp.name !== step.name)
+                throw new NonDeterminismError(runId, seq, step.name, cp.name);
+            if (cp.status === 'completed') {
+                outputs[i] = cp.output;
+                continue;
+            }
+            try {
+                outputs[i] = await this.callRemote(runId, seq, step, inputs[i], undefined, opts.transport, undefined, opts.admission, runNamespace, group);
+            }
+            catch (err) {
+                if (err instanceof WorkflowSuspended) {
+                    inFlight += 1;
+                    if (err.wakeAt !== undefined)
+                        wakeAt = Math.min(wakeAt ?? err.wakeAt, err.wakeAt);
+                }
+                else if (err instanceof RemoteStepError) {
+                    const failure = {
+                        index: i,
+                        id: stepId(runId, seq),
+                        error: err.message,
+                        ...codeAndDetails(err),
+                    };
+                    if (opts.mode === 'failFast')
+                        throw new GatherError([failure], inputs.length);
+                    failures.push(failure);
+                }
+                else {
+                    throw err;
+                }
+            }
+        }
+        const batch = inFlight > 0
+            ? []
+            : opts.concurrency === undefined
+                ? unsent
+                : unsent.slice(0, opts.concurrency);
+        if (batch.length > 0) {
+            await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, runNamespace);
+        }
+        if (inFlight + batch.length > 0) {
+            throw new WorkflowSuspended(wakeAt, undefined, undefined, [{ from: first, to: last }]);
+        }
+        if (opts.settle) {
+            const failed = new Map(failures.map((f) => [f.index, f]));
+            // `outputs` has a hole at every failed index, which `map` would skip.
+            return Array.from(outputs, (value, i) => {
+                const f = failed.get(i);
+                if (!f)
+                    return { status: 'fulfilled', value };
+                const { index: _index, ...rejected } = f;
+                return { status: 'rejected', ...rejected };
+            });
+        }
+        if (failures.length > 0)
+            throw new GatherError(failures, inputs.length);
+        return outputs;
+    }
+    /** Write the `pending` checkpoints of a step fan's new steps, then dispatch them. */
+    async sendStepFan(runId, positions, step, inputs, batch, group, opts, runNamespace) {
+        const token = stepGroup(step, runNamespace);
+        await this.ensureRoutable(token, step.requires);
+        const input = (i) => step.input ? step.input.parse(inputs[i]) : inputs[i];
+        const validInputs = new Map(batch.map((i) => [i, input(i)]));
+        const enqueuedAt = new Date();
+        for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
+            await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map((i) => {
+                const seq = positions[i];
+                return this.store.saveCheckpoint({
+                    runId,
+                    seq,
+                    name: step.name,
+                    kind: 'remote',
+                    stepId: stepId(runId, seq),
+                    status: 'pending',
+                    input: validInputs.get(i),
+                    attempts: 1,
+                    workerGroup: token,
+                    parallelGroup: group,
+                    enqueuedAt,
+                    startedAt: enqueuedAt,
+                    finishedAt: enqueuedAt,
+                });
+            }));
+        }
+        for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
+            await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map(async (i) => {
+                const seq = positions[i];
+                await this.dispatchRemoteTask({
+                    runId,
+                    seq,
+                    name: step.name,
+                    stepId: stepId(runId, seq),
+                    group: token,
+                    input: validInputs.get(i),
+                    priority: opts.admission?.priority,
+                    attempt: 1,
+                    transport: opts.transport,
+                });
+                this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
+            }));
+        }
+    }
     /**
      * Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
      * dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
@@ -4147,18 +4553,34 @@ export class WorkflowEngine {
         return true;
     }
     async completeRemoteResult(result) {
-        const runId = await this.settleRemoteCheckpoint(result);
-        if (runId === undefined)
+        const cp = await this.settleRemoteCheckpoint(result);
+        if (cp === undefined)
             return;
+        if (await this.stepFanStillRunning(cp, result))
+            return;
+        const runId = result.runId;
         this.trackEffect(this.resumeImplicit(runId).catch((err) => {
             console.warn(`[adonis-durable] background resume of run ${runId} failed (the run recovers on its next reconcile wake): ${err instanceof Error ? err.message : String(err)}`);
         }));
     }
+    /**
+     * The join barrier of a step fan: a result for one of its steps resumes the parent only once no
+     * step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
+     * retry needs the parent to schedule it). Without it every result would replay the whole parent,
+     * which makes a fan of N steps cost O(N²) checkpoint reads. A redelivered result goes through it
+     * too: its re-driven resume is only owed once the fan has settled.
+     */
+    async stepFanStillRunning(cp, result) {
+        const fan = parseStepFanGroup(cp.parallelGroup);
+        if (!fan || result.status === 'failed')
+            return false;
+        return ((await this.countPendingSteps(result.runId, fan.firstSeq, fan.firstSeq + fan.size - 1)) > 0);
+    }
     /**
      * The durable half of {@link completeRemoteResult}: settle `result` onto its checkpoint (or re-drive
-     * the resume when a redelivered result finds it already settled). Returns the run to resume, or
-     * `undefined` when no resume is wanted (no checkpoint, or a late result for a terminal run —
-     * terminal is terminal).
+     * the resume when a redelivered result finds it already settled). Returns the step's checkpoint as
+     * read when its run should be resumed, or `undefined` when no resume is wanted (no checkpoint, or a
+     * late result for a terminal run — terminal is terminal).
      */
     async settleRemoteCheckpoint(result) {
         const cp = await this.store.getCheckpoint(result.runId, result.seq);
@@ -4186,7 +4608,7 @@ export class WorkflowEngine {
         // explicit `requeue`/`durable:retry`, never to a stray result.
         if (cp.status !== 'pending') {
             if (!terminal)
-                return result.runId;
+                return cp;
             return undefined;
         }
         // A result settling this step frees its flow-control slot (no-op if it wasn't queued). Released
@@ -4229,7 +4651,7 @@ export class WorkflowEngine {
             finishedAt,
         });
         if (!claimed)
-            return terminal ? undefined : result.runId;
+            return terminal ? undefined : cp;
         await this.releaseQueueSlot(cp);
         this.emit({
             type: result.status === 'completed' ? 'step.completed' : 'step.failed',
@@ -4242,7 +4664,7 @@ export class WorkflowEngine {
             queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
             durationMs: finishedAt.getTime() - startedAt.getTime(),
         });
-        return result.runId;
+        return cp;
     }
     /**
      * A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they
@@ -4470,17 +4892,4 @@ export class RemoteStepError extends Error {
 }
 /** Checkpoint names that record a parent→child edge: an awaited child, a folded group of them, a spawn. */
 const CHILD_EDGE_PREFIXES = ['signal:child:', FOLDED_PREFIX, 'spawn:'];
-/** The child run ids a parent's {@link CHILD_EDGE_PREFIXES} checkpoints point at. */
-function childIdsOf(checkpoints) {
-    const ids = [];
-    for (const cp of checkpoints) {
-        if (cp.name.startsWith('signal:child:'))
-            ids.push(cp.name.slice('signal:child:'.length));
-        if (cp.name.startsWith(FOLDED_PREFIX))
-            ids.push(...foldedChildIds(cp));
-        if (cp.name.startsWith('spawn:') && typeof cp.output === 'string')
-            ids.push(cp.output);
-    }
-    return ids;
-}
 //# sourceMappingURL=engine.js.map