@adonis-agora/durable 0.54.1 → 0.54.3

package/dist/src/engine.js

@@ -407,6 +407,14 @@ export class WorkflowEngine {
     turnSettledFans = new Map();
     /** The kept wake reasons each executing turn on this instance serves (see {@link execute}). */
     turnServedWakes = new Map();
+    /**
+     * `<runId>:<seq>` of the steps (not of a step fan) whose result this instance settled and whose
+     * run it then resumed, with when, kept for a lease. That resume runs a turn after any this instance
+     * is executing, or asks the lease holder for one, so the re-check after an in-process turn
+     * ({@link recheckWaitSeqs}) has no wake to recover for such a step: resuming again would only
+     * replay the run once more.
+     */
+    settledHere = new Map();
     /** {@link sweepBufferedSignals}: the id the pass in progress resumes after, `undefined` between passes. */
     bufferedSweepCursor;
     bufferedSweepActive = false;
@@ -544,6 +552,7 @@ export class WorkflowEngine {
                 this.latest.get(run.workflow))?.singleton,
         });
         this.pool.bind(async (result) => {
+            this.heartbeatWrites.delete(result.stepId);
             // In-memory path (a `timeoutMs` step awaiting on THIS instance): resolve its pending promise.
             const waiter = this.pending.get(result.stepId);
             if (waiter) {
@@ -1158,9 +1167,35 @@ export class WorkflowEngine {
         if (this.heartbeatPersists.size > 1024)
             this.heartbeatPersists.clear();
         this.heartbeatPersists.set(id, { at: now, hadProgress: beat.progress !== undefined });
-        void this.store
-            .recordStepHeartbeat(beat.runId, beat.seq, new Date(beat.at ?? now), beat.progress)
-            .catch(() => undefined);
+        this.heartbeatWrites.set(id, {
+            runId: beat.runId,
+            seq: beat.seq,
+            at: new Date(beat.at ?? now),
+            progress: beat.progress,
+        });
+        if (!this.heartbeatWriter) {
+            this.heartbeatWriter = this.writeHeartbeats().finally(() => {
+                this.heartbeatWriter = undefined;
+            });
+            this.trackEffect(this.heartbeatWriter);
+        }
+    }
+    /** Beats {@link persistHeartbeat} let through and {@link writeHeartbeats} has not written yet, by
+     *  stepId: a newer beat of the same step replaces the queued one, a step's result drops it. */
+    heartbeatWrites = new Map();
+    heartbeatWriter;
+    /**
+     * Write the queued beats one at a time, oldest first, until none is left. One write in flight keeps
+     * a burst of beats (every step of a wide fan sends one when a worker picks it up) from taking the
+     * store's connections away from the turns and results, and a step that finishes while its beat
+     * waits never has it written: nothing reads a settled step's liveness.
+     */
+    async writeHeartbeats() {
+        const record = this.store.recordStepHeartbeat?.bind(this.store);
+        for (const [id, beat] of this.heartbeatWrites) {
+            this.heartbeatWrites.delete(id);
+            await record?.(beat.runId, beat.seq, beat.at, beat.progress).catch(() => undefined);
+        }
     }
     /**
      * Declare this process a worker: flush consumer subscriptions parked by a transport whose
@@ -2127,13 +2162,19 @@ export class WorkflowEngine {
      * delivery's own resume no-oped against this turn's still-held lease (see the WorkflowSuspended
      * catch in runExecution). Deferred one macrotask so the caller's `finally` releases the lease
      * first; tracked so `drain()` waits for the re-driven turn's writes.
+     *
+     * With `settledHereResumes`, a step in {@link settledHere} needs no re-drive: the turn that
+     * suspended on it read its snapshot before the settle, so the settle's resume queued behind that
+     * turn or asked the lease holder for one. A remote workflow's decision is applied after the turn
+     * that computed it ended, so the settle's resume may already have run, and it passes `false`.
      */
-    recheckWaitSeqs(runId, seqs) {
+    recheckWaitSeqs(runId, seqs, settledHereResumes = false) {
         this.trackEffect((async () => {
             await new Promise((r) => setTimeout(r, 0));
             for (const seq of seqs) {
-                const cp = await this.store.getCheckpoint(runId, seq).catch(() => null);
-                if (cp && (cp.status === 'completed' || cp.status === 'failed')) {
+                if (await this.checkpointSettled(runId, seq).catch(() => false)) {
+                    if (settledHereResumes && this.settledHere.has(`${runId}:${seq}`))
+                        return;
                     await this.resumeImplicit(runId).catch(() => undefined);
                     return;
                 }
@@ -2150,20 +2191,31 @@ export class WorkflowEngine {
         this.trackEffect((async () => {
             await new Promise((r) => setTimeout(r, 0));
             for (const fan of fans) {
-                const pending = await this.countPendingSteps(runId, fan.from, fan.to).catch(() => 1);
-                if (pending === 0) {
+                const pending = await this.stepFanHasPending(runId, fan.from, fan.to).catch(() => true);
+                if (!pending) {
                     await this.resumeImplicit(runId).catch(() => undefined);
                     return;
                 }
             }
         })());
     }
-    /** {@link StateStore.countCheckpoints} of `pending` steps, or a JS count on a store without it. */
-    async countPendingSteps(runId, from, to) {
+    /** Whether the checkpoint at `seq` is settled: a `COUNT` when the store has one, else the row. */
+    async checkpointSettled(runId, seq) {
+        if (this.store.countCheckpoints) {
+            return (await this.store.countCheckpoints(runId, seq, seq, ['completed', 'failed'])) > 0;
+        }
+        const cp = await this.store.getCheckpoint(runId, seq);
+        return cp?.status === 'completed' || cp?.status === 'failed';
+    }
+    /** Whether any step of a step fan's seq range is still `pending`. */
+    async stepFanHasPending(runId, from, to) {
+        if (this.store.hasCheckpoints) {
+            return this.store.hasCheckpoints(runId, from, to, ['pending']);
+        }
         if (this.store.countCheckpoints) {
-            return this.store.countCheckpoints(runId, from, to, ['pending']);
+            return (await this.store.countCheckpoints(runId, from, to, ['pending'])) > 0;
         }
-        return (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= from && c.seq <= to && c.status === 'pending').length;
+        return (await this.store.listCheckpoints(runId)).some((c) => c.seq >= from && c.seq <= to && c.status === 'pending');
     }
     /**
      * Track an in-flight execution so {@link drain} can wait for it.
@@ -4379,7 +4431,7 @@ export class WorkflowEngine {
         const routePlan = await this.planRoute(registered.convention ? conventionGroup(run.workflow, run.namespace) : remote.group, registered.requires);
         if (routePlan?.status === 'blocked')
             return this.parkBlocked(run, routePlan);
-        const checkpoints = await this.store.listCheckpoints(run.id);
+        const checkpoints = await this.replayCheckpoints(run.id);
         const handoff = liveHandoff(run.id, checkpoints);
         if (handoff) {
             // A run that handed its failure to an onError handler only follows the handler from here on.
@@ -4943,6 +4995,42 @@ export class WorkflowEngine {
     /** Earliest clock time a step's lease is worth LOOKING at again (`stepId` → clock ms) — the read
      *  throttle for {@link rearmStepLease}, the sibling of {@link heartbeatPersists}. */
     stepLeaseRearms = new Map();
+    /**
+     * {@link stepLeaseRearms} for the steps of a fan this instance sent, by run: one entry per sent
+     * block of consecutive seqs instead of one per step, since they share the lease stamped at send.
+     */
+    fanLeaseRearms = new Map();
+    noteFanLease(runId, seqs, lease) {
+        if (this.remoteRedispatchMs == null || seqs.length === 0)
+            return;
+        let from = seqs[0];
+        let to = from;
+        for (const seq of seqs) {
+            if (seq < from)
+                from = seq;
+            if (seq > to)
+                to = seq;
+        }
+        // A gap would cover steps sent earlier, under an older lease.
+        if (to - from + 1 !== seqs.length)
+            return;
+        if (this.fanLeaseRearms.size > 1024)
+            this.fanLeaseRearms.clear();
+        const blocks = this.fanLeaseRearms.get(runId) ?? [];
+        blocks.push({ from, to, lookAgainAt: lease - Math.ceil(this.remoteRedispatchMs / 2) });
+        this.fanLeaseRearms.set(runId, blocks);
+    }
+    fanLeaseLookAgainAt(runId, seq, now) {
+        const blocks = this.fanLeaseRearms.get(runId);
+        if (!blocks)
+            return undefined;
+        const live = blocks.filter((b) => now < b.lookAgainAt);
+        if (live.length === 0)
+            this.fanLeaseRearms.delete(runId);
+        else if (live.length < blocks.length)
+            this.fanLeaseRearms.set(runId, live);
+        return live.find((b) => seq >= b.from && seq <= b.to)?.lookAgainAt;
+    }
     /**
      * A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
      * that is still working on it is never presumed dead and double-dispatched. The durable counterpart
@@ -4989,7 +5077,7 @@ export class WorkflowEngine {
             return;
         const id = stepId(runId, seq);
         const now = this.clock();
-        const lookAgainAt = this.stepLeaseRearms.get(id);
+        const lookAgainAt = this.stepLeaseRearms.get(id) ?? this.fanLeaseLookAgainAt(runId, seq, now);
         if (lookAgainAt !== undefined && now < lookAgainAt)
             return;
         const cp = await this.store.getCheckpoint(runId, seq);
@@ -5397,7 +5485,7 @@ export class WorkflowEngine {
         // Read once at execution start: a checkpoint written AFTER this snapshot (the signal/timer/child
         // this resume wakes on, or one written later in this same execution) is absent from the map, and
         // the ctx falls back to the live store for any absent seq — so replay semantics are unchanged.
-        const snapshot = await this.store.listCheckpoints(run.id);
+        const snapshot = await this.replayCheckpoints(run.id);
         const replay = new Map();
         for (const cp of snapshot)
             replay.set(cp.seq, cp);
@@ -5490,7 +5578,7 @@ export class WorkflowEngine {
                 // all with reconcileMs: 0). Re-check the seqs this suspension waits on and re-drive if one
                 // already settled.
                 if (settled.status === 'suspended' && err.waitSeqs?.length) {
-                    this.recheckWaitSeqs(run.id, err.waitSeqs);
+                    this.recheckWaitSeqs(run.id, err.waitSeqs, true);
                 }
                 if (settled.status === 'suspended' && err.stepFans?.length) {
                     this.recheckStepFans(run.id, err.stepFans);
@@ -6311,7 +6399,11 @@ export class WorkflowEngine {
                 ? unsent
                 : unsent.slice(0, opts.concurrency);
         if (batch.length > 0) {
-            await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, runNamespace);
+            const lease = await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, runNamespace);
+            if (lease !== undefined) {
+                const at = this.stepLeaseWakeAt(lease);
+                wakeAt = Math.min(wakeAt ?? at, at);
+            }
         }
         if (inFlight + batch.length > 0) {
             throw new WorkflowSuspended(wakeAt, undefined, undefined, [{ from: first, to: last }]);
@@ -6343,7 +6435,8 @@ export class WorkflowEngine {
     /**
      * Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
      * {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the batch,
-     * where the store and transport have them.
+     * where the store and transport have them. With the lost-dispatch self-heal on, each step is
+     * written with its lease already stamped, and the lease is returned for the run to park on.
      */
     async sendStepFan(runId, positions, step, inputs, batch, group, opts, runNamespace) {
         const token = stepGroup(step, runNamespace);
@@ -6355,6 +6448,7 @@ export class WorkflowEngine {
         for (const i of batch)
             validInputs.set(i, await input(i));
         const enqueuedAt = new Date();
+        const lease = this.remoteRedispatchMs == null ? undefined : this.clock() + this.remoteRedispatchMs;
         const checkpoints = batch.map((i) => {
             const seq = positions[i];
             return {
@@ -6371,6 +6465,7 @@ export class WorkflowEngine {
                 enqueuedAt,
                 startedAt: enqueuedAt,
                 finishedAt: enqueuedAt,
+                ...(lease !== undefined ? { wakeAt: lease } : {}),
             };
         });
         if (this.store.saveCheckpoints) {
@@ -6398,6 +6493,10 @@ export class WorkflowEngine {
         for (const cp of checkpoints) {
             this.emit({ type: 'step.started', runId, seq: cp.seq, name: step.name, kind: 'remote' });
         }
+        if (lease !== undefined) {
+            this.noteFanLease(runId, checkpoints.map((cp) => cp.seq), lease);
+        }
+        return lease;
     }
     /**
      * Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
@@ -6418,6 +6517,13 @@ export class WorkflowEngine {
             attempt: task.attempt,
         }, task.transport);
     }
+    /** A turn's read of its run's checkpoints: {@link StateStore.listCheckpointsForReplay}, or every
+     *  column through {@link StateStore.listCheckpoints} on a store without it. */
+    replayCheckpoints(runId) {
+        return this.store.listCheckpointsForReplay
+            ? this.store.listCheckpointsForReplay(runId)
+            : this.store.listCheckpoints(runId);
+    }
     /**
      * Complete a durable remote step from its worker result and resume the run — runs on whichever
      * instance receives the result (the dispatching one may be gone), so the run is crash/scale-safe.
@@ -6500,8 +6606,28 @@ export class WorkflowEngine {
             return;
         if (await this.stepFanStillRunning(cp, result, queuedIn))
             return;
+        if (!parseStepFanGroup(cp.parallelGroup))
+            this.noteSettledHere(result.runId, [cp.seq]);
         this.resumeInBackground(result.runId, fanWakeOf(cp, result));
     }
+    /**
+     * Record `seqs` of `runId` in {@link settledHere}, dropping the entries older than a lease. A key is
+     * re-inserted at the end, so the map stays in the order its keys were noted and pruning stops at
+     * the first fresh entry.
+     */
+    noteSettledHere(runId, seqs) {
+        const now = this.clock();
+        for (const seq of seqs) {
+            const key = `${runId}:${seq}`;
+            this.settledHere.delete(key);
+            this.settledHere.set(key, now);
+        }
+        for (const [key, at] of this.settledHere) {
+            if (now - at <= this.leaseMs)
+                break;
+            this.settledHere.delete(key);
+        }
+    }
     resumeInBackground(runId, fan) {
         this.trackEffect(this.resumeImplicit(runId, fan).catch((err) => {
             console.warn(`[adonis-durable] background resume of run ${runId} failed (the run recovers on its next reconcile wake): ${err instanceof Error ? err.message : String(err)}`);
@@ -6565,20 +6691,11 @@ export class WorkflowEngine {
                 resumable.push({ cp: read.value, result });
         });
         const finishedAt = new Date();
-        const writes = pending.map(({ index, cp }) => {
-            const result = results[index];
-            return {
-                ...cp,
-                status: result.status,
-                output: result.status === 'completed' ? result.output : cp.output,
-                error: result.error,
-                events: result.events ?? cp.events,
-                startedAt: result.startedAt ? new Date(result.startedAt) : cp.startedAt,
-                finishedAt,
-            };
-        });
         // Claimed, not blindly written: a duplicate delivery may have settled a step first.
-        const claims = await this.savePendingCheckpoints(writes);
+        const claims = await this.settlePendingCheckpoints(pending.map(({ index, cp }) => ({
+            cp,
+            settlement: settlementOf(cp, results[index], finishedAt),
+        })));
         for (const [k, claim] of claims.entries()) {
             const { index, cp } = pending[k];
             const result = results[index];
@@ -6624,9 +6741,14 @@ export class WorkflowEngine {
             if (fan)
                 highest.set(fan.firstSeq, Math.max(highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
         }
+        // The one resume below replays every step of the batch.
+        const singles = resumable
+            .filter(({ cp }) => !parseStepFanGroup(cp.parallelGroup))
+            .map(({ cp }) => cp.seq);
         for (const { cp, result } of resumable) {
             const fan = parseStepFanGroup(cp.parallelGroup);
             if (!fan || result.status === 'failed') {
+                this.noteSettledHere(runId, singles);
                 this.resumeInBackground(runId);
                 return outcomes;
             }
@@ -6635,10 +6757,11 @@ export class WorkflowEngine {
                 const lastSeq = fan.firstSeq + fan.size - 1;
                 closed = queuedIn(fan.firstSeq, lastSeq)
                     ? Promise.resolve(true)
-                    : this.countPendingSteps(runId, fan.firstSeq, lastSeq).then((n) => n > 0, () => false);
+                    : this.stepFanHasPending(runId, fan.firstSeq, lastSeq).catch(() => false);
                 barriers.set(fan.firstSeq, closed);
             }
             if (!(await closed)) {
+                this.noteSettledHere(runId, singles);
                 this.resumeInBackground(runId, {
                     first: fan.firstSeq,
                     seq: highest.get(fan.firstSeq) ?? cp.seq,
@@ -6648,6 +6771,29 @@ export class WorkflowEngine {
         }
         return outcomes;
     }
+    /**
+     * Settle pending steps, each only while its row is still `pending`; per entry, whether it wrote
+     * or the error that kept it from writing. Through {@link StateStore.settleCheckpointsIfPending}
+     * when the store has it: one call, or one per entry when that call rejects (it then wrote
+     * nothing). Else each step's whole row is written through {@link savePendingCheckpoints}.
+     */
+    async settlePendingCheckpoints(entries) {
+        const settle = this.store.settleCheckpointsIfPending?.bind(this.store);
+        if (!settle) {
+            return this.savePendingCheckpoints(entries.map(({ cp, settlement }) => ({ ...cp, ...settlement })));
+        }
+        const settlements = entries.map((e) => e.settlement);
+        try {
+            const wrote = await settle(settlements);
+            return settlements.map((_, i) => wrote[i] === true);
+        }
+        catch (error) {
+            if (settlements.length === 1)
+                return [{ error }];
+        }
+        const settled = await Promise.allSettled(settlements.map((s) => settle([s])));
+        return settled.map((r) => r.status === 'fulfilled' ? r.value[0] === true : { error: r.reason });
+    }
     /**
      * {@link savePendingCheckpoint} for checkpoints at distinct seqs: one
      * {@link StateStore.saveCheckpointsIfPending} call when the store has it, else one write each.
@@ -6682,7 +6828,17 @@ export class WorkflowEngine {
         // Left to the result of this fan queued behind this one (see `completeRemoteResults`).
         if (queuedIn(fan.firstSeq, lastSeq))
             return true;
-        return (await this.countPendingSteps(result.runId, fan.firstSeq, lastSeq)) > 0;
+        return this.stepFanHasPending(result.runId, fan.firstSeq, lastSeq);
+    }
+    /**
+     * {@link StateStore.getCheckpointsForSettle}, when the store can use what it leaves out: only
+     * with {@link StateStore.settleCheckpointsIfPending}, since a settle without it (and the salvage
+     * of a late success onto a failed run) writes back the whole row it read.
+     */
+    settleReader() {
+        return this.store.settleCheckpointsIfPending
+            ? this.store.getCheckpointsForSettle?.bind(this.store)
+            : undefined;
     }
     /**
      * The checkpoints of one run at `seqs`, each as its own outcome: one
@@ -6690,6 +6846,19 @@ export class WorkflowEngine {
      * call rejecting falls back to one read each, so a failing read fails only its own entry.
      */
     async readCheckpoints(runId, seqs) {
+        const forSettle = this.settleReader();
+        if (forSettle) {
+            if (seqs.length > 1) {
+                try {
+                    const bySeq = new Map((await forSettle(runId, seqs)).map((cp) => [cp.seq, cp]));
+                    return seqs.map((seq) => ({ status: 'fulfilled', value: bySeq.get(seq) ?? null }));
+                }
+                catch {
+                    // Fall through: one by one.
+                }
+            }
+            return Promise.allSettled(seqs.map(async (seq) => (await forSettle(runId, [seq]))[0] ?? null));
+        }
         if (seqs.length > 1 && this.store.getCheckpoints) {
             try {
                 const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [cp.seq, cp]));
@@ -6708,7 +6877,10 @@ export class WorkflowEngine {
      * late result for a terminal run — terminal is terminal).
      */
     async settleRemoteCheckpoint(result) {
-        const cp = await this.store.getCheckpoint(result.runId, result.seq);
+        const forSettle = this.settleReader();
+        const cp = forSettle
+            ? ((await forSettle(result.runId, [result.seq]))[0] ?? null)
+            : await this.store.getCheckpoint(result.runId, result.seq);
         if (!cp)
             return undefined;
         const run = await this.store.getRun(result.runId);
@@ -6751,31 +6923,25 @@ export class WorkflowEngine {
         if (run && (run.status === 'failed' || run.status === 'dead')) {
             await this.releaseQueueSlot(cp);
             if (result.status === 'completed') {
-                await this.store.saveCheckpoint({
-                    ...cp,
-                    status: 'completed',
-                    output: result.output,
-                    events: result.events ?? cp.events,
-                    startedAt: result.startedAt ? new Date(result.startedAt) : cp.startedAt,
-                    finishedAt: new Date(),
-                });
+                const salvage = { ...settlementOf(cp, result, new Date()), error: cp.error };
+                if (this.store.settleCheckpointsIfPending) {
+                    await this.store.settleCheckpointsIfPending([salvage]);
+                }
+                else {
+                    await this.store.saveCheckpoint({ ...cp, ...salvage });
+                }
             }
             return undefined;
         }
         const finishedAt = new Date();
-        const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
+        const settlement = settlementOf(cp, result, finishedAt);
+        const startedAt = settlement.startedAt;
         // Claimed, not blindly written: another delivery settled it first, which is the redelivery case
         // above — re-drive the resume rather than settle twice.
-        const claimed = await this.savePendingCheckpoint({
-            ...cp,
-            status: result.status,
-            output: result.status === 'completed' ? result.output : cp.output,
-            error: result.error,
-            events: result.events ?? cp.events,
-            startedAt,
-            finishedAt,
-        });
-        if (!claimed)
+        const [claim] = await this.settlePendingCheckpoints([{ cp, settlement }]);
+        if (typeof claim === 'object')
+            throw claim.error;
+        if (!claim)
             return terminal ? undefined : cp;
         await this.releaseQueueSlot(cp);
         this.emit({
@@ -7060,6 +7226,19 @@ export class WorkflowEngine {
 function conventionGroup(name, namespace) {
     return tenantGroup(sanitizeQueueToken(name), namespace);
 }
+/** What `result` settles onto its step's pending checkpoint `cp`. */
+function settlementOf(cp, result, finishedAt) {
+    return {
+        runId: cp.runId,
+        seq: cp.seq,
+        status: result.status,
+        output: result.status === 'completed' ? result.output : cp.output,
+        error: result.error,
+        events: result.events ?? cp.events,
+        startedAt: result.startedAt ? new Date(result.startedAt) : cp.startedAt,
+        finishedAt,
+    };
+}
 /** Key of an inline remote turn's liveness reset in `heartbeatResets` (step keys are `runId:seq`). */
 function turnBeatKey(runId) {
     return `turn:${runId}`;