@adonis-agora/durable 0.56.2 → 0.56.3

package/dist/src/engine.js

@@ -122,6 +122,9 @@ const STEP_FAN_CHUNK = 200;
  * waiting for every checkpoint of a wide fan to commit.
  */
 const STEP_FAN_SEND_ROUND = 2000;
+/** Most step events one control-plane message carries (see {@link WorkflowEngine.broadcastEvents}). */
+const STEP_EVENTS_PER_BROADCAST = 500;
+const isStepEvent = (event) => event.type.startsWith('step.');
 /** The default {@link WorkflowEngineDeps.timerBatchSize}. */
 const DEFAULT_TIMER_BATCH = 500;
 /** {@link WorkflowEngine.sweepBufferedSignals}: rows read per page. */
@@ -161,6 +164,24 @@ const LEASE_RECHECK_SLACK_MS = 100;
  * turn; the calls already in flight are not re-dispatched (their pending checkpoints dedupe them).
  */
 const REMOTE_TURN_REDRIVE_MS = 1_000;
+/**
+ * The most entries a per-step in-memory throttle/fallback map keeps (heartbeat writes, lease
+ * re-arms, admitted queues). Each entry is a few dozen bytes; the cap is there so a long-lived pod
+ * that hears the whole fleet's beats cannot grow without bound, while staying well above the number
+ * of steps a fleet keeps in flight, so the throttles keep working at scale.
+ */
+const THROTTLE_MAP_CAP = 16_384;
+/** `map.set(key, value)` keeping insertion order as recency, evicting the oldest past {@link THROTTLE_MAP_CAP}. */
+function setBounded(map, key, value) {
+    map.delete(key);
+    map.set(key, value);
+    while (map.size > THROTTLE_MAP_CAP) {
+        const oldest = map.keys().next();
+        if (oldest.done)
+            break;
+        map.delete(oldest.value);
+    }
+}
 /** Fisher–Yates, in place. */
 function shuffle(items) {
     for (let i = items.length - 1; i > 0; i -= 1) {
@@ -327,7 +348,8 @@ export class WorkflowEngine {
     admission;
     /** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
     queueWaiters = new Map();
-    /** Which queue a dispatched step took a slot from, by stepId — so the result can release it. */
+    /** Which queue a dispatched step took a slot from, by stepId — so the result can release it.
+     *  Bounded (see {@link setBounded}): the checkpoint's `queue` is the durable source. */
     stepQueue = new Map();
     /** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
     inflight = new Map();
@@ -621,6 +643,11 @@ export class WorkflowEngine {
                 // `at` may be a string after JSON transit (Redis) — normalize back to a Date.
                 this.deliver(markRemoteEvent({ ...msg.event, at: new Date(msg.event.at) }));
             }
+            else if (msg.kind === 'events') {
+                for (const event of msg.events) {
+                    this.deliver(markRemoteEvent({ ...event, at: new Date(event.at) }));
+                }
+            }
             else if (msg.kind === 'cancel') {
                 this.notifyCancelled(msg.runId);
             }
@@ -1156,8 +1183,11 @@ export class WorkflowEngine {
      * One exemption: a beat CARRYING progress punches through when the last persisted one had none —
      * the automatic pickup beat lands first (no payload) and must not shadow the handler's first real
      * progress for a whole window. Optional store capability: no `recordStepHeartbeat` → no-op.
-     * Failures are swallowed (a lost liveness update never harms the run). The throttle map is cleared
-     * wholesale past 1024 entries — a rare extra write, never a leak.
+     * Failures are swallowed (a lost liveness update never harms the run). The throttle map stays in
+     * write order: entries whose window has passed are dropped from its front (they no longer throttle
+     * anything), and past {@link THROTTLE_MAP_CAP} the oldest goes — a rare extra write, never a leak.
+     * It used to be cleared wholesale past 1024 entries, which a fleet with more steps beating than that
+     * (every engine pod hears every beat) hit every few seconds, turning the throttle off.
      */
     persistHeartbeat(beat) {
         const id = beat.stepId;
@@ -1170,9 +1200,15 @@ export class WorkflowEngine {
             now - last.at < this.heartbeatPersistThrottleMs &&
             !upgradesToProgress)
             return;
-        if (this.heartbeatPersists.size > 1024)
-            this.heartbeatPersists.clear();
+        // Re-insert at the back so the map stays ordered by write time, then drop the expired front.
+        this.heartbeatPersists.delete(id);
         this.heartbeatPersists.set(id, { at: now, hadProgress: beat.progress !== undefined });
+        for (const [oldId, entry] of this.heartbeatPersists) {
+            if (now - entry.at < this.heartbeatPersistThrottleMs &&
+                this.heartbeatPersists.size <= THROTTLE_MAP_CAP)
+                break;
+            this.heartbeatPersists.delete(oldId);
+        }
         this.heartbeatWrites.set(id, {
             runId: beat.runId,
             seq: beat.seq,
@@ -1335,12 +1371,51 @@ export class WorkflowEngine {
     emit(event) {
         const full = { ...event, at: new Date() };
         this.deliver(full);
-        if (this.controlPlane) {
-            void this.controlPlane
-                .publishControl({ kind: 'event', event: full, from: this.instanceId })
-                .catch(() => {
-                // control-plane delivery is best-effort observability; never break execution
-            });
+        if (!this.controlPlane)
+            return;
+        this.eventsToBroadcast.push(full);
+        if (!this.broadcastingEvents)
+            void this.broadcastEvents();
+    }
+    /** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
+    eventsToBroadcast = [];
+    broadcastingEvents = false;
+    /**
+     * Publish the queued events in the order they were emitted. A run's own events go out one per
+     * message, as every instance expects them. Step events (a wide fan emits two per step) that queued
+     * up while a publish was in flight go out together in one `events` message, so a fan of 40,000
+     * steps costs the control plane a few hundred messages to every instance instead of 80,000.
+     */
+    async broadcastEvents() {
+        this.broadcastingEvents = true;
+        try {
+            while (this.eventsToBroadcast.length > 0 && this.controlPlane) {
+                const first = this.eventsToBroadcast[0];
+                let msg;
+                if (isStepEvent(first)) {
+                    let n = 1;
+                    while (n < STEP_EVENTS_PER_BROADCAST &&
+                        n < this.eventsToBroadcast.length &&
+                        isStepEvent(this.eventsToBroadcast[n])) {
+                        n += 1;
+                    }
+                    const events = this.eventsToBroadcast.splice(0, n);
+                    msg =
+                        events.length === 1
+                            ? { kind: 'event', event: first, from: this.instanceId }
+                            : { kind: 'events', events, from: this.instanceId };
+                }
+                else {
+                    this.eventsToBroadcast.shift();
+                    msg = { kind: 'event', event: first, from: this.instanceId };
+                }
+                await this.controlPlane.publishControl(msg).catch(() => {
+                    // control-plane delivery is best-effort observability; never break execution
+                });
+            }
+        }
+        finally {
+            this.broadcastingEvents = false;
         }
     }
     /** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
@@ -1726,7 +1801,12 @@ export class WorkflowEngine {
             //  (b) an old pending REMOTE step whose worker heartbeat is silent — dispatched and lost.
             // A long `ctx.sleep` (wakeAt in the future) and a beating long step are NOT stalled.
             let stalePending = null;
-            for (const cp of await this.store.listCheckpoints(run.id)) {
+            // Only pending checkpoints can be stale: read just those where the store can filter, not the
+            // run's whole history (a wide fan's thousands of settled rows, payloads included).
+            const pendingCheckpoints = this.store.queryCheckpoints
+                ? await this.store.queryCheckpoints(run.id, { statuses: ['pending'] })
+                : await this.store.listCheckpoints(run.id);
+            for (const cp of pendingCheckpoints) {
                 if (cp.kind !== 'remote' || cp.status !== 'pending')
                     continue;
                 const ageMs = now - cp.enqueuedAt.getTime();
@@ -5066,7 +5146,8 @@ export class WorkflowEngine {
      * (with a 30-minute window: two writes an hour), independent of beat frequency. The in-memory mark
      * is a pure optimisation — cold on a fresh instance, or on the one that didn't dispatch the step —
      * and the durable half-spent check re-derives the same answer from the checkpoint, so nothing
-     * depends on it surviving. Cleared wholesale past 1024 entries: a rare extra read, never a leak.
+     * depends on it surviving. Bounded at {@link THROTTLE_MAP_CAP} entries, oldest first: a rare extra
+     * read, never a leak.
      *
      * NEVER REJECTS. This runs on the transport's SERIAL heartbeat handler, so a store hiccup thrown out
      * of here would take the whole fleet's beat loop with it — and every lease would then lapse at once,
@@ -5104,15 +5185,13 @@ export class WorkflowEngine {
         const halfWindow = Math.ceil(this.remoteRedispatchMs / 2);
         // More than half the window left: nothing to buy, and nothing to look at until it is half spent.
         if (cp.wakeAt != null && cp.wakeAt > now + halfWindow) {
-            this.stepLeaseRearms.set(id, cp.wakeAt - halfWindow);
+            setBounded(this.stepLeaseRearms, id, cp.wakeAt - halfWindow);
             return;
         }
         const renewed = now + this.remoteRedispatchMs;
         const previous = cp.wakeAt;
         await this.store.saveCheckpoint({ ...cp, wakeAt: renewed });
-        if (this.stepLeaseRearms.size > 1024)
-            this.stepLeaseRearms.clear();
-        this.stepLeaseRearms.set(id, renewed - halfWindow);
+        setBounded(this.stepLeaseRearms, id, renewed - halfWindow);
         // If the run is parked EXACTLY on this step's old lease, carry its wake forward too — otherwise a
         // beating worker costs one wasted turn per renewal window. Only on an exact match: any other
         // `wakeAt` belongs to some other timer (a sleep, a sibling's shorter lease) and moving it would
@@ -6265,7 +6344,9 @@ export class WorkflowEngine {
                 throw new WorkflowSuspended(decision.retryAt);
             }
             this.queueWaiters.get(queue)?.delete(runId);
-            this.stepQueue.set(id, queue);
+            // Bounded: the result (and so the delete) may land on another pod, which releases the slot
+            // from the checkpoint's durable `queue` — this map is only the legacy fallback.
+            setBounded(this.stepQueue, id, queue);
             admittedQueue = queue;
         }
         // Optional runtime schema: validate only when the def carries one (a bare `@Step()` / string-name