@adonis-agora/durable 0.45.0 → 0.46.0

package/dist/src/engine.js

@@ -1,18 +1,21 @@
 import { InMemoryAdmissionBackend } from './admission.js';
 import { backoffDelay, MAX_BACKOFF_MS } from './backoff.js';
 import { instantCheckpoint, lostStepEvent, stepCheckpoint } from './checkpoints.js';
-import { assertConcurrency, concurrencyTag, resolveConcurrency, } from './concurrency.js';
+import { isFailedCompletion } from './completion.js';
+import { assertConcurrency, concurrencyTag, countRuns, resolveConcurrency, } from './concurrency.js';
 import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispatch-routing.js';
 import { parseDuration } from './duration.js';
 import { Entities } from './entities.js';
 import { ContinueAsNew, NonDeterminismError, RemoteStepTimeout, WorkflowSuspended, } from './errors.js';
 import { EventAccumulators } from './event-accumulators.js';
 import { eventMatches, eventMatchOf, eventPrefix } from './events.js';
+import { FOLDED_PREFIX, fanOfWaiter, foldedChildIds } from './fan-join.js';
 import { isNonEmptyRunScope, TERMINAL_RUN_STATUSES } from './interfaces.js';
 import { asHeartbeat } from './observability-scope.js';
 import { breakpointToken, stepId } from './protocol.js';
 import { RemoteWorkflowExecutor } from './remote-workflow-executor.js';
 import { scanRunValueFacets } from './run-value-facets.js';
+import { listSignalWaitersOfRuns } from './run-waiting.js';
 import { scheduleWindow } from './scheduler.js';
 import { ScheduleClient } from './schedules.js';
 import { SingletonGate } from './singleton-gate.js';
@@ -291,6 +294,7 @@ export class WorkflowEngine {
         this.schedules = new ScheduleClient(this.store, {
             start: (workflow, input, runId, opts) => this.start(workflow, input, runId, opts),
             getRun: (runId) => this.store.getRun(runId),
+            isDraining: () => this.draining,
         }, { namespace: this.namespace, clock: this.clock });
         this.stalledAfterMs = Math.max(60_000, deps.stalledAfterMs ?? 900_000);
         this.trackStepStart = deps.trackStepStart ?? true;
@@ -878,10 +882,11 @@ export class WorkflowEngine {
      * dispatcher the run executes off-instance, so only the durable persist is guaranteed. A start that
      * throws is handed to `onStartError` (a child delivers it to its waiting parent) instead of vanishing.
      */
-    handoffRun(workflow, input, runId, opts, onStartError) {
+    handoffRun(workflow, input, runId, opts, onStartError, onAlreadySettled) {
         this.trackEffect((async () => {
             // Defer one microtask: never reentrantly resume the parent that is still suspending/settling.
             await Promise.resolve();
+            let result;
             try {
                 if (this.usesInProcessDispatch) {
                     // Persist WITHOUT the fire-and-forget dispatch, then drive the pickup ourselves so this
@@ -889,7 +894,7 @@ export class WorkflowEngine {
                     // stays tracked until the run has entered `inflight` and settled. `runOne` no-ops (returns
                     // null) on replay (the run already exists) or while draining (the run is left `pending` for
                     // recovery) — the durable persist above is what matters on those paths.
-                    const result = await this.startCore(workflow, input, runId, opts, false);
+                    result = await this.startCore(workflow, input, runId, opts, false);
                     // Guard-free pickup: the continuation/child is in-flight work `drain()` is waiting on, so it
                     // runs even while draining (a `runOne` here would no-op on `this.draining`, stranding it
                     // `pending` after its persist — persisted-but-not-processed). Mirrors a signal-driven resume.
@@ -899,12 +904,19 @@ export class WorkflowEngine {
                 else {
                     // Custom dispatcher owns execution (possibly on another pod); guarantee the persist + its
                     // normal dispatch/broadcast. The in-process rolled-back-transaction hazard doesn't apply.
-                    await this.startCore(workflow, input, runId, opts, true);
+                    result = await this.startCore(workflow, input, runId, opts, true);
                 }
             }
             catch (err) {
                 if (onStartError)
                     onStartError(err);
+                return;
+            }
+            // A new run is never terminal, so a terminal result is a run that already existed.
+            if (onAlreadySettled &&
+                result &&
+                TERMINAL_RUN_STATUSES.includes(result.status)) {
+                await onAlreadySettled(result);
             }
         })());
     }
@@ -1356,6 +1368,19 @@ export class WorkflowEngine {
         this.postSettle.add(tracked);
         void tracked.finally(() => this.postSettle.delete(tracked));
     }
+    /**
+     * Stop picking up new work without waiting: recovery, timer, pending, schedule and timeout sweeps
+     * (and their per-run lock loops) start nothing once this is called, and {@link runOne} refuses
+     * broker-delivered runs. Runs already started finish. {@link drain} calls it first; the worker loop
+     * calls it the moment a stop is requested, so a tick still in flight stops leasing a backlog.
+     */
+    beginDrain() {
+        this.draining = true;
+    }
+    /** True once {@link beginDrain} (or {@link drain}) has been called. */
+    isDraining() {
+        return this.draining;
+    }
     /**
      * Graceful shutdown: stop picking up new runs (recovery/timer become no-ops) and wait for
      * in-flight executions to settle, up to `timeoutMs`. Call from your app's shutdown hook so a
@@ -1376,7 +1401,7 @@ export class WorkflowEngine {
      * `stepId`, idempotent by the durable contract; a completed step short-circuits).
      */
     async drain(timeoutMs = 10_000) {
-        this.draining = true;
+        this.beginDrain();
         if (this.inflight.size === 0 && this.postSettle.size === 0)
             return;
         const timer = new Promise((resolve) => {
@@ -1423,6 +1448,8 @@ export class WorkflowEngine {
      */
     async sweepTimeouts(now = this.clock()) {
         for (const reg of new Set(this.latest.values())) {
+            if (this.draining)
+                return;
             if (reg.executionTimeoutMs == null)
                 continue;
             // Push the age predicate into the store: only candidates already past the LATEST version's
@@ -1437,6 +1464,8 @@ export class WorkflowEngine {
                 size: 200,
             });
             for (const run of inflight) {
+                if (this.draining)
+                    return;
                 // Time out against the version the run STARTED on when that registration is known — a fleet
                 // mid-deploy must not sweep old-version runs by the newest version's (possibly shorter)
                 // timeout. Falls back to the latest registration's timeout for an unknown version.
@@ -1500,6 +1529,8 @@ export class WorkflowEngine {
             return (await this.store.listIncompleteRuns(this.namespace)).filter((r) => r.lockedUntil === undefined || r.lockedUntil <= nowMs);
         });
         for (const listed of incomplete) {
+            if (this.draining)
+                break; // stop requested mid-sweep
             // A live worker renews its lease, so an acquirable lease means the run is genuinely orphaned
             // (its worker crashed). Skip the ones still owned.
             const acquired = await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
@@ -1652,8 +1683,8 @@ export class WorkflowEngine {
         // - a `failed` checkpoint becomes retryable-now (attempts 0, wake immediately, `retryable`
         //   forced back on): the durable-retry machinery re-dispatches it as a fresh first attempt;
         // - an awaited-child `signal:child:` checkpoint holding a FAILURE completion returns to its
-        //   live `running` placeholder, so replay re-registers the `child:<id>` waiter — a separately
-        //   retried child's completion (already buffered, or still to come) then resumes this run.
+        //   live `running` placeholder with its `child:<id>` waiter put back — a separately retried
+        //   child's completion (already buffered, or still to come) then resumes this run.
         //   Retry child and parent in either order; signal buffering makes it converge.
         if (run.status === 'failed' || run.status === 'dead') {
             const isFailureCompletion = (v) => typeof v === 'object' && v !== null && v.ok === false;
@@ -1670,16 +1701,15 @@ export class WorkflowEngine {
                     cp.name.startsWith('signal:child:') &&
                     cp.status === 'completed' &&
                     isFailureCompletion(cp.output)) {
-                    await this.store.saveCheckpoint({ ...cp, status: 'running', output: undefined });
-                    // CASCADE: retrying only the parent is useless in a live engine — replay re-registers
-                    // the `child:<id>` waiter and the reconciler re-delivers the child's still-FAILED
-                    // terminal state within seconds, re-failing the parent with the same error. Requeue the
-                    // failed child too so the pair converges: whichever finishes the handshake last (waiter
-                    // registered vs completion buffered) resumes the parent.
+                    // CASCADE: retrying only the parent is useless in a live engine — the reconciler
+                    // re-delivers the child's still-FAILED terminal state within seconds, re-failing the
+                    // parent with the same error. Requeue the failed child too so the pair converges:
+                    // whichever finishes the handshake last (waiter registered vs completion buffered)
+                    // resumes the parent.
                     //
                     // Exception: a SUCCESS already buffered on this token (a completed `~retry~` fix of the
-                    // child — see notifyParent's origin delivery) is the outcome the replay will consume, so
-                    // re-running the failed origin would be pure waste. Peek = take + re-buffer.
+                    // child — see notifyParent's origin delivery) is the outcome, so re-running the failed
+                    // origin would be pure waste: settle the checkpoint with it right here.
                     const childId = cp.name.slice('signal:child:'.length);
                     // Drain-and-restore keeps relative order (a partial take+re-buffer would rotate it).
                     const parked = [];
@@ -1689,11 +1719,27 @@ export class WorkflowEngine {
                             break;
                         parked.push(buffered.payload);
                     }
-                    for (const payload of parked)
-                        await this.store.bufferSignal(`child:${childId}`, payload);
-                    const bufferedSuccess = parked.some((payload) => typeof payload === 'object' &&
+                    const successAt = parked.findIndex((payload) => typeof payload === 'object' &&
                         payload !== null &&
                         payload.ok === true);
+                    const success = successAt === -1 ? undefined : parked.splice(successAt, 1)[0];
+                    for (const payload of parked)
+                        await this.store.bufferSignal(`child:${childId}`, payload);
+                    const bufferedSuccess = success !== undefined;
+                    if (bufferedSuccess) {
+                        await this.store.saveCheckpoint({ ...cp, output: success });
+                    }
+                    else {
+                        // Back to a live `running` placeholder. Its waiter goes first: replay reads a
+                        // placeholder as proof its waiter exists and never re-puts it.
+                        await this.store.putSignalWaiter({
+                            token: `child:${childId}`,
+                            runId,
+                            seq: cp.seq,
+                            parallelGroup: cp.parallelGroup,
+                        });
+                        await this.store.saveCheckpoint({ ...cp, status: 'running', output: undefined });
+                    }
                     if (!bufferedSuccess) {
                         const child = await this.store.getRun(childId);
                         if (child && (child.status === 'failed' || child.status === 'dead')) {
@@ -1865,6 +1911,9 @@ export class WorkflowEngine {
                 next += 1;
                 if (!run)
                     return;
+                // Re-check per run: a stop requested mid-tick must not lock/start the rest of the backlog.
+                if (this.draining)
+                    return;
                 const acquired = await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs);
                 if (!acquired)
                     continue;
@@ -1978,8 +2027,60 @@ export class WorkflowEngine {
             // for an ordinary (non-fan) signal/child await.
             parallelGroup: waiter.parallelGroup,
         }));
+        if (await this.fanStillWaiting(waiter, payload)) {
+            return { runId: waiter.runId, status: 'suspended' };
+        }
         return this.resumeImplicit(waiter.runId);
     }
+    /**
+     * The `ctx.all` join barrier: true when `waiter` was one child of a fan (or of one wave of it)
+     * whose siblings are still out, so waking the parent now would only replay the fan and re-suspend.
+     * Skipping that wake is what keeps a fan of N children at O(N) work instead of N replays of N items.
+     *
+     * Safe because the LAST sibling to settle always wakes the parent: each child consumes its own
+     * waiter before it asks, so whichever asks last sees none left. A failed child of a failFast fan
+     * always wakes it. Anything that is not a `ctx.all` waiter, or a store that cannot answer the
+     * question cheaply, wakes the parent as before.
+     */
+    async fanStillWaiting(waiter, payload) {
+        if (!this.store.hasSignalWaiter)
+            return false;
+        const fan = fanOfWaiter(waiter);
+        if (!fan)
+            return false;
+        if (fan.failFast && isFailedCompletion(payload))
+            return false;
+        return this.store.hasSignalWaiter(waiter.runId, fan.waiterPrefix);
+    }
+    /**
+     * Re-drive a run that just suspended on awaited children (a `ctx.all` fan or wave, a `ctx.child`)
+     * which, by now, have settled.
+     *
+     * A wake that finds the run's lease held is dropped. So if a child settles while the parent is
+     * still executing (the turn that dispatched it, or one woken by an earlier child), that wake is
+     * spent, and the turn suspends on a snapshot that predates the child. With the `ctx.all` barrier
+     * that wake was the only one the fan would get, and nothing would wake the run until the
+     * `reconcileMs` sweep.
+     *
+     * Runs after the turn released its lease, so any child that settles from here on wakes the run
+     * itself. Only re-drives for a group whose checkpoints are all settled, which the next turn is
+     * then guaranteed to see, so it cannot loop. A group that still has a waiter is skipped without
+     * reading its checkpoints.
+     */
+    async redriveSettledChildren(runId, fan) {
+        for (const wait of fan.waits) {
+            if (await this.store.hasSignalWaiter?.(runId, wait.waiterPrefix))
+                continue;
+            const checkpoints = this.store.listCheckpointsByNamePrefix
+                ? await this.store.listCheckpointsByNamePrefix(runId, [wait.checkpointPrefix])
+                : (await this.store.listCheckpoints(runId)).filter((cp) => cp.name.startsWith(wait.checkpointPrefix));
+            const group = checkpoints.filter((cp) => cp.name.startsWith(wait.checkpointPrefix));
+            if (group.length > 0 && group.every((cp) => cp.status === 'completed')) {
+                await this.resumeImplicit(runId);
+                return;
+            }
+        }
+    }
     /**
      * Deliver an external signal to the run waiting on `token`, resume it with `payload`, and return
      * the run result — or null if no run is (or ends up) waiting for it.
@@ -2092,7 +2193,23 @@ export class WorkflowEngine {
                 ok: false,
                 error: `child workflow "${workflow}" failed to start: ${message}`,
             });
-        });
+        }, (settled) => this.redeliverSettledChild(childId, settled));
+    }
+    /**
+     * A child start that found the child already terminal: its completion may never reach the waiter
+     * the parent just put (it was buffered before that waiter existed, or lost between the child's
+     * terminal write and its notify). Deliver it again, preferring a buffered copy, so the awaiting
+     * parent does not need a run lookup per child to notice.
+     */
+    async redeliverSettledChild(childId, settled) {
+        const token = `child:${childId}`;
+        const buffered = await this.store.takeBufferedSignal(token);
+        const completion = buffered
+            ? buffered.payload
+            : settled.status === 'completed'
+                ? { ok: true, value: settled.output }
+                : { ok: false, error: settled.error?.message ?? settled.status };
+        await this.signal(token, completion);
     }
     /**
      * Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
@@ -2207,32 +2324,23 @@ export class WorkflowEngine {
      * so a completed parent (or a completed awaited child) would otherwise drop out of the tree. The
      * `signal:child:<id>` checkpoint (the placeholder written when the parent first awaits the child,
      * overwritten as completed/failed when it settles) persists across completion, so we read both and
-     * dedupe — the edge stays stable for a finished run, not just a live one.
+     * dedupe — the edge stays stable for a finished run, not just a live one. A settled `ctx.all`
+     * group folded into one `fan:` checkpoint stands for every child in it.
      */
     async getRunChildren(parentRunId) {
         const childIds = new Set();
-        // Targeted per-run lookup when the store has it (an indexed `WHERE run_id`); the prefix scan
-        // fallback walks EVERY `child:` waiter row for one parent.
-        const waiters = this.store.listSignalWaitersByRunIds
-            ? await this.store.listSignalWaitersByRunIds([parentRunId])
-            : await this.store.listSignalWaiters('child:');
-        for (const w of waiters) {
-            if (w.runId === parentRunId && w.token.startsWith('child:')) {
+        // The parent's own waiters only (an indexed `WHERE run_id` when the store has it).
+        for (const w of await listSignalWaitersOfRuns(this.store, [parentRunId])) {
+            if (w.token.startsWith('child:'))
                 childIds.add(w.token.slice('child:'.length));
-            }
         }
-        // Targeted read: only the `signal:child:` / `spawn:` checkpoints, not the whole history. Falls
-        // back to a full listCheckpoints + in-JS prefix scan for a custom store that omits the method.
-        const prefixes = ['signal:child:', 'spawn:'];
+        // Targeted read: only the child-edge checkpoints, not the whole history. Falls back to a full
+        // listCheckpoints + in-JS prefix scan for a custom store that omits the method.
         const childCheckpoints = this.store.listCheckpointsByNamePrefix
-            ? await this.store.listCheckpointsByNamePrefix(parentRunId, prefixes)
-            : (await this.store.listCheckpoints(parentRunId)).filter((cp) => prefixes.some((p) => cp.name.startsWith(p)));
-        for (const cp of childCheckpoints) {
-            if (cp.name.startsWith('signal:child:'))
-                childIds.add(cp.name.slice('signal:child:'.length));
-            if (cp.name.startsWith('spawn:') && typeof cp.output === 'string')
-                childIds.add(cp.output);
-        }
+            ? await this.store.listCheckpointsByNamePrefix(parentRunId, CHILD_EDGE_PREFIXES)
+            : (await this.store.listCheckpoints(parentRunId)).filter((cp) => CHILD_EDGE_PREFIXES.some((p) => cp.name.startsWith(p)));
+        for (const id of childIdsOf(childCheckpoints))
+            childIds.add(id);
         return [...childIds];
     }
     /**
@@ -2251,14 +2359,15 @@ export class WorkflowEngine {
      * stamp. Prefers the store's indexed method; falls back to filtering the full scan for a custom
      * store without it.
      */
-    async listSignalWaitersByRunIds(runIds) {
-        if (runIds.length === 0)
-            return [];
-        if (this.store.listSignalWaitersByRunIds) {
-            return this.store.listSignalWaitersByRunIds(runIds);
-        }
-        const ids = new Set(runIds);
-        return (await this.store.listSignalWaiters('')).filter((w) => ids.has(w.runId));
+    listSignalWaitersByRunIds(runIds) {
+        return listSignalWaitersOfRuns(this.store, runIds);
+    }
+    /**
+     * How many runs match `query`: an exact `COUNT` through the store's `countRuns`, or the length of
+     * the full listing for a store without one.
+     */
+    countRuns(query) {
+        return countRuns(this.store, query);
     }
     /** The worker groups this engine dispatches to: every registered remote workflow's group, plus any
      *  `extra` the caller declares. Local-step groups (a group consumed by in-process `@DurableStep`
@@ -3211,6 +3320,7 @@ export class WorkflowEngine {
         const replay = new Map();
         for (const cp of snapshot)
             replay.set(cp.seq, cp);
+        let suspendedOnFan;
         const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow);
         // Every settle below is `return await`ed, never a bare `return this.settleRun(...)`: inside
         // `try … finally` a bare return runs the `finally` — which RELEASES THE LEASE — as soon as the
@@ -3322,6 +3432,8 @@ export class WorkflowEngine {
                 if (settled.status === 'suspended' && err.waitSeqs?.length) {
                     this.recheckWaitSeqs(run.id, err.waitSeqs);
                 }
+                if (settled.status === 'suspended')
+                    suspendedOnFan = err.fan;
                 return settled;
             }
             const error = {
@@ -3362,6 +3474,10 @@ export class WorkflowEngine {
             catch (releaseErr) {
                 console.warn(`[adonis-durable] releasing the lease of run ${run.id} failed (it expires on its own and recovery re-drives): ${releaseErr instanceof Error ? releaseErr.message : String(releaseErr)}`);
             }
+            if (suspendedOnFan) {
+                const fan = suspendedOnFan;
+                this.trackEffect(this.redriveSettledChildren(run.id, fan));
+            }
         }
     }
     /**
@@ -4330,4 +4446,19 @@ export class RemoteStepError extends Error {
         this.stepError = stepError;
     }
 }
+/** Checkpoint names that record a parent→child edge: an awaited child, a folded group of them, a spawn. */
+const CHILD_EDGE_PREFIXES = ['signal:child:', FOLDED_PREFIX, 'spawn:'];
+/** The child run ids a parent's {@link CHILD_EDGE_PREFIXES} checkpoints point at. */
+function childIdsOf(checkpoints) {
+    const ids = [];
+    for (const cp of checkpoints) {
+        if (cp.name.startsWith('signal:child:'))
+            ids.push(cp.name.slice('signal:child:'.length));
+        if (cp.name.startsWith(FOLDED_PREFIX))
+            ids.push(...foldedChildIds(cp));
+        if (cp.name.startsWith('spawn:') && typeof cp.output === 'string')
+            ids.push(cp.output);
+    }
+    return ids;
+}
 //# sourceMappingURL=engine.js.map