@adonis-agora/durable 0.40.0 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,7 +10,7 @@
10
10
  rel="stylesheet"
11
11
  />
12
12
  <title>durable · control plane</title>
13
- <script type="module" crossorigin src="/__DURABLE_DASHBOARD__/assets/index-yJslNGUY.js"></script>
13
+ <script type="module" crossorigin src="/__DURABLE_DASHBOARD__/assets/index-CeHc6GHq.js"></script>
14
14
  <link rel="stylesheet" crossorigin href="/__DURABLE_DASHBOARD__/assets/index-DU6cGcJz.css">
15
15
  </head>
16
16
  <body>
@@ -1,4 +1,19 @@
1
1
  import type { StepCheckpoint, StepError, StepEvent, StepKind } from './interfaces.js';
2
+ /**
3
+ * The step-event names the lost-dispatch self-heal stamps on a checkpoint's trail, so an operator (and
4
+ * the dashboard, which renders a step's events under it) can tell a re-drive apart from an ordinary
5
+ * failure retry WITHOUT a schema change: `step.redispatched` = the dispatched job was presumed lost
6
+ * (its lease lapsed with no result and no heartbeat) and the step was re-enqueued; `step.lost` = the
7
+ * re-drive bound (`remoteRedispatchMax`) ran out and the step was failed `remote_step_lost`.
8
+ */
9
+ export declare const REDISPATCHED_STEP_EVENT = "step.redispatched";
10
+ export declare const LOST_STEP_EVENT = "step.lost";
11
+ /**
12
+ * A `warn` step event recording a lost-dispatch re-drive (or the give-up at its bound). `attempts` is
13
+ * the dispatch count AFTER the transition, so the trail reads as the attempt an operator sees on the
14
+ * checkpoint.
15
+ */
16
+ export declare function lostStepEvent(at: number, attempts: number, outcome: 'redispatched' | 'lost'): StepEvent;
2
17
  /**
3
18
  * Build a *phased* step checkpoint — one with distinct enqueue/run/finish timestamps (a local step
4
19
  * that is running/completed/failed, or a pending remote/sleep step). Computes the deterministic
@@ -1 +1 @@
1
- {"version":3,"file":"checkpoints.d.ts","sourceRoot":"","sources":["../../src/checkpoints.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAQtF;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAC9B,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,GAAG,cAAc,CAoBjB;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,GAAG,cAAc,CAiBjB"}
1
+ {"version":3,"file":"checkpoints.d.ts","sourceRoot":"","sources":["../../src/checkpoints.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAGtF;;;;;;GAMG;AACH,eAAO,MAAM,uBAAuB,sBAAsB,CAAC;AAC3D,eAAO,MAAM,eAAe,cAAc,CAAC;AAE3C;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,cAAc,GAAG,MAAM,GAC/B,SAAS,CAUX;AAOD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAC9B,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,GAAG,cAAc,CAoBjB;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,GAAG,cAAc,CAiBjB"}
@@ -1,4 +1,28 @@
1
1
  import { stepId } from './protocol.js';
2
+ /**
3
+ * The step-event names the lost-dispatch self-heal stamps on a checkpoint's trail, so an operator (and
4
+ * the dashboard, which renders a step's events under it) can tell a re-drive apart from an ordinary
5
+ * failure retry WITHOUT a schema change: `step.redispatched` = the dispatched job was presumed lost
6
+ * (its lease lapsed with no result and no heartbeat) and the step was re-enqueued; `step.lost` = the
7
+ * re-drive bound (`remoteRedispatchMax`) ran out and the step was failed `remote_step_lost`.
8
+ */
9
+ export const REDISPATCHED_STEP_EVENT = 'step.redispatched';
10
+ export const LOST_STEP_EVENT = 'step.lost';
11
+ /**
12
+ * A `warn` step event recording a lost-dispatch re-drive (or the give-up at its bound). `attempts` is
13
+ * the dispatch count AFTER the transition, so the trail reads as the attempt an operator sees on the
14
+ * checkpoint.
15
+ */
16
+ export function lostStepEvent(at, attempts, outcome) {
17
+ return {
18
+ at,
19
+ level: 'warn',
20
+ name: outcome === 'lost' ? LOST_STEP_EVENT : REDISPATCHED_STEP_EVENT,
21
+ message: outcome === 'lost'
22
+ ? `dispatch lost and the re-drive bound is spent after ${attempts} dispatch(es) — failing the step`
23
+ : `re-dispatched after a lost dispatch (attempt ${attempts}): the step's lease lapsed with no result and no heartbeat`,
24
+ };
25
+ }
2
26
  /** Drop an empty events array to `undefined` (the repeated `events?.length ? events : undefined`). */
3
27
  function nonEmptyEvents(events) {
4
28
  return events && events.length > 0 ? events : undefined;
@@ -1 +1 @@
1
- {"version":3,"file":"checkpoints.js","sourceRoot":"","sources":["../../src/checkpoints.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,sGAAsG;AACtG,SAAS,cAAc,CAAC,MAA+B;IACrD,OAAO,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,CAiB9B;IACC,OAAO;QACL,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC;QAChC,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,UAAU,EAAE,CAAC,CAAC,UAAU;KACzB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,CASjC;IACC,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;IACtB,OAAO;QACL,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,WAAW;QAC/B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,EAAE;QACd,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;KACf,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"checkpoints.js","sourceRoot":"","sources":["../../src/checkpoints.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,mBAAmB,CAAC;AAC3D,MAAM,CAAC,MAAM,eAAe,GAAG,WAAW,CAAC;AAE3C;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,EAAU,EACV,QAAgB,EAChB,OAAgC;IAEhC,OAAO;QACL,EAAE;QACF,KAAK,EAAE,MAAM;QACb,IAAI,EAAE,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,uBAAuB;QACpE,OAAO,EACL,OAAO,KAAK,MAAM;YAChB,CAAC,CAAC,uDAAuD,QAAQ,kCAAkC;YACnG,CAAC,CAAC,gDAAgD,QAAQ,4DAA4D;KAC3H,CAAC;AACJ,CAAC;AAED,sGAAsG;AACtG,SAAS,cAAc,CAAC,MAA+B;IACrD,OAAO,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,CAiB9B;IACC,OAAO;QACL,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC;QAChC,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,UAAU,EAAE,CAAC,CAAC,UAAU;KACzB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,CASjC;IACC,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;IACtB,OAAO;QACL,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,WAAW;QAC/B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,EAAE;QACd,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;KACf,CAAC;AACJ,CAAC"}
@@ -1,5 +1,5 @@
1
1
  /** Keep in sync with this package's `version` in package.json. */
2
- export declare const VERSION = "0.40.0";
2
+ export declare const VERSION = "0.41.0";
3
3
  export type { AccessDeniedInfo, AccessDeniedPageOptions, AccessDeniedReason, } from './access_denied_page.js';
4
4
  export { CONSOLE as ACCESS_DENIED_CONSOLE, escapeHtml, renderAccessDeniedPage, resolveAccessDeniedPage, } from './access_denied_page.js';
5
5
  export type { AuthMode, DashboardAuthOptions, LoginHook, LoginOutcome, ResolvedDashboardAuth, SessionHook, SessionOutcome, } from './auth.js';
@@ -1,5 +1,5 @@
1
1
  /** Keep in sync with this package's `version` in package.json. */
2
- export const VERSION = '0.40.0';
2
+ export const VERSION = '0.41.0';
3
3
  // The built-in "access denied" page (what a browser sees on a refused page navigation).
4
4
  export { CONSOLE as ACCESS_DENIED_CONSOLE, escapeHtml, renderAccessDeniedPage, resolveAccessDeniedPage, } from './access_denied_page.js';
5
5
  // Built-in `dashboardAuth` login screen (optional; opt-in via `config/durable_dashboard.ts`).
@@ -1002,10 +1002,87 @@ export declare class WorkflowEngine {
1002
1002
  */
1003
1003
  private reconcileWakeAt;
1004
1004
  private runRemoteExecution;
1005
+ /**
1006
+ * A polyglot workflow's re-emitted `call` whose checkpoint already exists: decide whether the step is
1007
+ * still out for delivery, or whether its dispatch was LOST and has to be re-driven. Returns the
1008
+ * deadline to park the run on (so the next decision lands exactly when it's due), or `undefined` when
1009
+ * there is nothing to wait for.
1010
+ *
1011
+ * WHY THIS EXISTS. A worker process killed mid-step (an OOM kill, exit 137 — a pod's memory ceiling,
1012
+ * not a bug in anyone's code) takes its in-flight job with it, and the transport cannot always put it
1013
+ * back: the `bullmq` transport's terminal-failure bridge (`worker.on('failed')`) only exists inside a
1014
+ * JS consumer, so a first-class Python fleet (see `docs/python.mdx`) has none; the `queue` transport's
1015
+ * reclaim sweep gives up past `maxStalledCount`, and never runs at all on an adapter without
1016
+ * `recoverStalledJobs`. NOTHING then settles the checkpoint. It stays `pending` — "out for delivery" —
1017
+ * and the `applyCommands` idempotency guard, which exists so a `ctx.gather_calls` fan-out doesn't
1018
+ * double-dispatch its still-in-flight siblings on every partial resume, skips it on every later turn.
1019
+ * The run wakes on the reconcile sweep, dispatches a turn, has the call re-emitted, skips it again,
1020
+ * and sleeps. Forever — `pending`, `attempts = 1`, `wakeAt` NULL.
1021
+ *
1022
+ * THE POLICY (mirrors {@link callRemote}'s `ctx.step` self-heal, which this path silently did not
1023
+ * share — so `remoteRedispatchMs` simply did not apply to a fan-out):
1024
+ * - a SETTLED checkpoint always wins: a step that completed just before the crash is never re-run;
1025
+ * - `remoteRedispatchMs` unset (the default) keeps the by-design "re-suspend, never re-dispatch";
1026
+ * - otherwise the pending checkpoint carries a LEASE in `wakeAt` — a result or a step heartbeat
1027
+ * ({@link rearmStepLease}) renews it, so a live worker still holding the step is never
1028
+ * double-dispatched — and only a LAPSED lease re-dispatches, bounded by `remoteRedispatchMax`;
1029
+ * - past the bound the step is failed `remote_step_lost` and the failure enters the run's history
1030
+ * (see {@link remoteHistory}), so the workflow's own error path surfaces it on the next turn
1031
+ * instead of the engine looping. The run is parked to wake immediately for exactly that turn.
1032
+ */
1033
+ private redriveLostCall;
1034
+ /**
1035
+ * The run `wakeAt` a step lease should park on: the lease itself, but never LATER than the
1036
+ * {@link reconcileMs} safety net would have woken the run anyway. A long lease (a fan-out step that
1037
+ * legitimately runs for half an hour) must not stretch the orphan sweep's cadence for this run — an
1038
+ * early wake just re-parks on the un-lapsed lease, which is cheap and convergent.
1039
+ */
1040
+ private stepLeaseWakeAt;
1041
+ /** Earliest clock time a step's lease is worth LOOKING at again (`stepId` → clock ms) — the read
1042
+ * throttle for {@link rearmStepLease}, the sibling of {@link heartbeatPersists}. */
1043
+ private readonly stepLeaseRearms;
1044
+ /**
1045
+ * A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
1046
+ * that is still working on it is never presumed dead and double-dispatched. The durable counterpart
1047
+ * of the in-memory `heartbeatResets` rearm (which only exists on the instance that dispatched a
1048
+ * `timeoutMs` step). No-op unless the lost-dispatch self-heal is on — without `remoteRedispatchMs`
1049
+ * nothing ever re-dispatches, so there is no lease to defend.
1050
+ *
1051
+ * THROTTLED, for the same reason {@link persistHeartbeat} is: beats arrive every few seconds from a
1052
+ * hot batch loop, and defending a lease must not turn into a per-beat SELECT + UPDATE storm. The
1053
+ * throttle is the lease itself rather than a fixed interval — a renewal is only WORTH a write once
1054
+ * the window is half spent, because it buys at most one window either way. So:
1055
+ * - a beat that finds more than half the window still on the clock writes nothing, and records the
1056
+ * half-spent mark so subsequent beats skip even the `getCheckpoint` READ until then;
1057
+ * - a beat at or past the half-spent mark renews, leaving a full half-window of headroom — the
1058
+ * lease can never lapse under a worker whose beats are anywhere near their normal cadence.
1059
+ *
1060
+ * That bounds the cost at ONE read + ONE write per half `remoteRedispatchMs` per in-flight step
1061
+ * (with a 30-minute window: two writes an hour), independent of beat frequency. The in-memory mark
1062
+ * is a pure optimisation — cold on a fresh instance, or on the one that didn't dispatch the step —
1063
+ * and the durable half-spent check re-derives the same answer from the checkpoint, so nothing
1064
+ * depends on it surviving. Cleared wholesale past 1024 entries: a rare extra read, never a leak.
1065
+ *
1066
+ * NEVER REJECTS. This runs on the transport's SERIAL heartbeat handler, so a store hiccup thrown out
1067
+ * of here would take the whole fleet's beat loop with it — and every lease would then lapse at once,
1068
+ * a far worse failure than the write storm the throttle removes. So the entire rearm is best-effort,
1069
+ * exactly like {@link persistHeartbeat}: a missed renewal costs at most one re-drive of a step that
1070
+ * is idempotent by contract, and because the in-memory mark is advanced only AFTER a successful
1071
+ * write, the very next beat retries the renewal rather than waiting out a window.
1072
+ */
1073
+ private rearmStepLease;
1074
+ /** {@link rearmStepLease}'s body — every durable read/write of the lease renewal, so the caller's one
1075
+ * guard covers all of them (the read, the checkpoint write, and the run's wake carry-forward). */
1076
+ private renewStepLease;
1005
1077
  /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
1006
1078
  private remoteHistory;
1007
1079
  /** Apply a turn's commands: persist recorded local steps, dispatch remote calls, schedule timers.
1008
- * Returns the earliest timer deadline to suspend on (or undefined — suspended on a result). */
1080
+ * Returns the earliest timer deadline to suspend on (or undefined — suspended on a result).
1081
+ *
1082
+ * `redriveLost` is set only by the `continue` path: a turn that PARKS on its blocking ops is the one
1083
+ * that may find a re-emitted `call` whose dispatch was lost and needs re-driving (see
1084
+ * {@link redriveLostCall}). A terminal turn (completed/failed/cancelled) applies its commands only to
1085
+ * record what already ran, and must never dispatch anything new. */
1009
1086
  private applyCommands;
1010
1087
  /** The run body, lease held + renewed by {@link execute}. */
1011
1088
  private runExecution;
@@ -1 +1 @@
1
- {"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,gBAAgB,EAA4B,MAAM,gBAAgB,CAAC;AAYjF,OAAO,EAAY,KAAK,YAAY,EAAE,MAAM,eAAe,CAAC;AAO5D,OAAO,EAAqB,KAAK,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAGnF,OAAO,KAAK,EACV,YAAY,EAEZ,cAAc,EACd,WAAW,EAGX,cAAc,EAEd,aAAa,EACb,aAAa,EACb,QAAQ,EACR,SAAS,EACT,SAAS,EACT,YAAY,EACZ,oBAAoB,EACpB,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,cAAc,EAEd,SAAS,EAET,eAAe,EAIf,SAAS,EACT,YAAY,EACZ,eAAe,EAEf,WAAW,EAEX,gBAAgB,EAChB,WAAW,EAEZ,MAAM,iBAAiB,CAAC;AAGzB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9C,OAAO,EAAE,KAAK,iBAAiB,EAAkB,MAAM,gBAAgB,CAAC;AAWxE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,WAAW,EAEjB,MAAM,mBAAmB,CAAC;AAE3B,KAAK,UAAU,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEzE,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B,gGAAgG;IAChG,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC5B,0FAA0F;IAC1F,gBAAgB,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;IAChD;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B;AAED,gGAAgG;AAChG,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,mGAAmG;IACnG,MAAM,EAAE,OAAO,CAAC;IAChB,8EAA8E;IAC9E,eAAe,EAAE,OAAO,CAAC;IACzB,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,kBAAkB,EAAE,MAAM,CAAC;IAC3B,6DAA6D;IAC7D,aAAa,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACvC;AAED,6GAA6G;AAC7G,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,WAAW,CAAC;IACjB,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAC;IACd,6FAA6F;IAC7F,WAAW,EAAE,OAAO,CAAC;IACrB,mFAAmF;IACnF,YAAY,EAAE;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,GAAG,IAAI,CAAC;CACV;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,MAAM,CAAC;IAChC,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AA6HD,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAClC;;;;OAIG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,GAAG,SAAS,CAAC;IAC1C;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC,oEAAoE;IACpE,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACnC,6FAA6F;IAC7F,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,mGAAmG;IACnG,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B;;;;OAIG;IACH,SAAS,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;IACzC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;;;;;;OAWG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,GAAG,SAAS,CAAC;IACrD;;;;OAIG;IACH,WAAW,CAAC,EAAE,CAAC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IACrD;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IAClE;;;;;;;;;OASG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC;IAC9F;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IAC/F;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACrC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,aAAa,GAAG,SAAS,CAAC;IAC1C;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAChD;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GAAG,SAAS,CAAC;IAChF;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC;AAED,6FAA6F;AAC7F,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,cAGC;CACF;AAED;;;;;GAKG;AACH,qBAAa,cAAc;;IACzB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAa;IACnC,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC;kFAC8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAwB;IAC1D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAA2B;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC;yFACqF;IACrF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAqB;IAC1D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAqB;IACxD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA0C;IACtE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAyC;IACtE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAA0D;IACnF,4IAA4I;IAC5I,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsE;IAChG,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAEZ;IAGd,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,cAAc,CAE3B;IACJ,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,0EAA0E;IAC1E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA6C;IAI9E,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IACzC,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAU;IAChD;0GACsG;IACtG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAmB;IAChD,sGAAsG;IACtG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IAEvC,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgB;IAC3C,0FAA0F;IAC1F,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyC;IACnE,kFAAkF;IAClF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyC;IAChE,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAChE,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAGpC,oGAAoG;IACpG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAoB;IACjD,wEAAwE;IACxE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,sFAAsF;IACtF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAiC;IACjE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAyB;IACtD,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsC;IACtE,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsC;IACxE,yFAAyF;IACzF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyC;IACvE,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAsC;IACvE,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAqB;IAKrD,mFAAmF;IACnF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,iGAAiG;IACjG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAkC;IAC/D,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,0FAA0F;IAC1F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyC;IAClE;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA+B;IAC1D,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,IAAI,EAAE,kBAAkB,EAqInC;IAED;;;;OAIG;IACH,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAE7D;IAED;;;;;;OAMG;IACH;;;;OAIG;IACH,cAAc,CAAC,SAAS,EAAE,SAAS,iBAAiB,EAAE,GAAG,IAAI,CAE5D;IAED;;;OAGG;IACH,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAEtD;IAED;;;;OAIG;IACG,aAAa,CAAC,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAwBvE;IAED;;;;OAIG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAYvD;IAED;;;;OAIG;IACG,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAOxF;IAED,iBAAiB,CAAC,SAAS,EAAE,SAAS,iBAAiB,EAAE,GAAG,IAAI,CAY/D;IAED,uGAAuG;IACvG,IAAI,mBAAmB,IAAI,SAAS,iBAAiB,EAAE,CAEtD;IAED,uFAAuF;IACvF,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAgC;IAEvE,sFAAsF;IACtF,qBAAqB,CAAC,QAAQ,EAAE,SAAS,sBAAsB,EAAE,GAAG,IAAI,CAEvE;IAED,6FAA6F;IAC7F,IAAI,uBAAuB,IAAI,SAAS,sBAAsB,EAAE,CAE/D;IAED,wFAAwF;IACxF,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhF;IAED,yGAAyG;IACzG,cAAc,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAE7E;IAED;;;;OAIG;IACH,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,IAAI,CAGxD;IAED,mGAAmG;IACnG,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,QAAQ,CACN,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,EAAE,EAAE,UAAU,EACd,IAAI,CAAC,EAAE;QACL,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC5B,SAAS,CAAC,EAAE,eAAe,GAAG,SAAS,CAAC;QACxC,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;QAC/C,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC;QACvE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC/B,UAAU,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;QAC1C,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAChC,8FAA8F;QAC9F,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;KAC7B,GACA,IAAI,CAuBN;IAED;;;;;OAKG;IACH,cAAc,CACZ,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,SAAS,CAAC,EAAE,eAAe,CAAC;QAC5B,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QACnC,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACzD,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;KACrB,GACA,IAAI,CAmBN;IAED;;;;;;;;OAQG;YACW,wBAAwB;IAYtC;;;;OAIG;IACH,aAAa,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAEvC;IAED;oCACgC;IAChC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA2D;IAE7F;;;;;;;;;OASG;IACH,OAAO,CAAC,gBAAgB;IAcxB;;;;;;OAMG;IACH,cAAc,IAAI,IAAI,CAErB;IAED,sEAAsE;IACtE,SAAS,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,IAAI,CAG9C;IAED;;;;;OAKG;IACH,GAAG,CAAC,WAAW,EAAE,eAAe,GAAG,MAAM,IAAI,CAM5C;IAED;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IAarB;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,IAAI,CAGtD;IAED;;;;;;OAMG;IACH,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI,CAGvD;IAED,OAAO,CAAC,UAAU;IAUlB;0FACsF;IACtF,OAAO,CAAC,IAAI;IAYZ;wGACoG;IACpG,OAAO,CAAC,OAAO;IAUT,KAAK,CAAC,CAAC,SAAS,aAAa,EACjC,QAAQ,EAAE,CAAC,EACX,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,EACzB,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC,SAAS,CAAC,CAAC;IAChB,KAAK,CAAC,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC,SAAS,CAAC,CAAC;IAatB;;;;;;OAMG;YACW,SAAS;IAkGvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,UAAU;IAmClB,mGAAmG;IACnG,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAEjD;IAED;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAEhD;IAED;;;;;OAKG;IACH,cAAc,CACZ,IAAI,EAAE,YAAY,EAClB,KAAK,EAAE,aAAa,EACpB,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAK7B;IAED,yGAAyG;IACzG,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAExD;IAED;;;;;OAKG;IACH;;;;;;;OAOG;IACH,SAAS,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,GAAG,MAAM,IAAI,CAG9D;IAED;;;OAGG;IACG,YAAY,CAAC,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAuDxE;IAED;;;;;OAKG;IACH,OAAO,CACL,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAClF,MAAM,IAAI,CAGZ;IAED;;;;;;;;;;OAUG;IACG,cAAc,CAAC,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,MAAM,CAAC,CAgChE;IAEK,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAoB9C;IAEK,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAsC9C;IAED;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAevB,qEAAqE;IACrE,OAAO,CAAC,KAAK;IAMb;;;;;;OAMG;IACH,OAAO,CAAC,WAAW;IAMnB;;;;;;;;;;;;;;;;;;OAkBG;IACG,KAAK,CAAC,SAAS,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB7C;IAED;;;OAGG;YACW,oBAAoB;IAclC;;;;OAIG;IACG,aAAa,CAAC,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwD7D;IAED;;;OAGG;IACG,iBAAiB,CAAC,KAAK,GAAE,MAAqB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAmD1E;IAED;;;;OAIG;YACW,aAAa;IAwB3B;;;;;;OAMG;IACG,eAAe,CAAC,KAAK,GAAE,MAAqB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAKxE;IAED;;;;;OAKG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAGrD;IAED;;;;;;OAMG;YACW,cAAc;IAiB5B;;;;;;OAMG;IACG,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CA6E3E;IAED;;;;;;;;;OASG;IACG,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,SAAS,GAAG;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI,CAAC,CA8B7F;IAED;;;;OAIG;IACG,cAAc,CAClB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,OAAO,EACd,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAQnC;IAED;;;;;OAKG;IACG,UAAU,CAAC,KAAK,GAAE,MAAqB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAKnE;IAED;;;;;;;;;;OAUG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAoD/F;IAED;;;OAGG;YACW,YAAY;IAiC1B;;;OAGG;IACH;;;;;;;;;;;;;;;;;;OAkBG;IACG,YAAY,CAChB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,IAAI,CAAC,EAAE;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,GACvC,OAAO,CAAC,MAAM,CAAC,CAsDjB;IAED;;;;OAIG;YACW,aAAa;IAkB3B;;;;;;;;;;;OAWG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAmBvE;IAED;;;;;;;OAOG;IACG,eAAe,CACnB,QAAQ,EAAE,WAAW,EACrB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,EAC5C,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAM5B;IAED;;;;OAIG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAK1F;IAED,6FAA6F;IACvF,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAEpF;IAED;;;;OAIG;IACH,OAAO,CAAC,YAAY;IA6BpB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,kBAAkB;IAkB1B;;;;;OAKG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CA+DtF;IAED;;;;;;;;OAQG;IACG,WAAW,CACf,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,EACvC,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9B,OAAO,CAAC,SAAS,EAAE,CAAC,CAQtB;IAED;;;;;OAKG;YACW,cAAc;IAS5B;;;;;;;;;;;OAWG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAyB3D;IAED;;;;;;;OAOG;IACH,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEzD;IAED;;;;OAIG;IACG,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAOzE;IAED;;;yGAGqG;IACrG,WAAW,CAAC,KAAK,GAAE,MAAM,EAAO,GAAG,MAAM,EAAE,CAM1C;IAED;;;;gEAI4D;IACtD,YAAY,CAAC,KAAK,GAAE,MAAM,EAAO,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAM/D;IAED;;;;;;;;OAQG;YACW,gBAAgB;IAsD9B;;;;;;OAMG;YACW,SAAS;IA4BvB,oGAAoG;YACtF,YAAY;IA8B1B,oGAAoG;YACtF,QAAQ;YA8BR,OAAO;IA0DrB;;;;;OAKG;IACH;;;;;;OAMG;YACW,SAAS;IAmFvB;;;;;OAKG;YACW,iBAAiB;IAc/B;;;;;OAKG;IACH,OAAO,CAAC,eAAe;YAKT,kBAAkB;IA2FhC,gGAAgG;YAClF,aAAa;IA0C3B;oGACgG;YAClF,aAAa;IA0N3B,6DAA6D;YAC/C,YAAY;IA+L1B;;;;;;;;;;;;;OAaG;YACW,eAAe;IAqE7B;;;;;;;;;;;;OAYG;YACW,oBAAoB;IAuElC;;;;;;OAMG;IACH,OAAO,CAAC,wBAAwB;IA6EhC;;wDAEoD;IACpD,OAAO,CAAC,UAAU;IAqDlB;;;;OAIG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAKvD;IAED;;;;OAIG;IACG,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAcxF;IAED;;;;OAIG;IACH,uBAAuB,CAAC,IAAI,EAC1B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,GAC9B,IAAI,CAEN;IAED;;;;;OAKG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,CAc7E;IAMD;;;0FAGsF;YACxE,kBAAkB;IAUhC;;;;;;OAMG;YACW,SAAS;IASvB;;;6EAGyE;YAC3D,cAAc;IAK5B;;;;;;;;OAQG;YACW,WAAW;IA2BzB;;qGAEiG;YACnF,cAAc;YAiBd,UAAU;IAgLxB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IA4B1B;;;;;;;;;;;;;;;;;;;;OAoBG;YACW,oBAAoB;IAclC;;;;;OAKG;YACW,sBAAsB;IA2EpC;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAQxB;;;;;;OAMG;YACW,gBAAgB;IAO9B,mGAAmG;YACrF,kBAAkB;IAwDhC,+FAA+F;YACjF,0BAA0B;IA0GxC,gGAAgG;YAClF,mBAAmB;IAgCjC;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;CAkD3B;AAED,8EAA8E;AAC9E,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAC3C,YAAY,SAAS,CAAC,EAAE,SAAS,EAIhC;CACF"}
1
+ {"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,gBAAgB,EAA4B,MAAM,gBAAgB,CAAC;AAYjF,OAAO,EAAY,KAAK,YAAY,EAAE,MAAM,eAAe,CAAC;AAO5D,OAAO,EAAqB,KAAK,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAGnF,OAAO,KAAK,EACV,YAAY,EAEZ,cAAc,EACd,WAAW,EAGX,cAAc,EAEd,aAAa,EACb,aAAa,EACb,QAAQ,EACR,SAAS,EACT,SAAS,EACT,YAAY,EACZ,oBAAoB,EACpB,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,cAAc,EAEd,SAAS,EAET,eAAe,EAIf,SAAS,EACT,YAAY,EACZ,eAAe,EAEf,WAAW,EAEX,gBAAgB,EAChB,WAAW,EAEZ,MAAM,iBAAiB,CAAC;AAGzB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9C,OAAO,EAAE,KAAK,iBAAiB,EAAkB,MAAM,gBAAgB,CAAC;AAWxE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,WAAW,EAEjB,MAAM,mBAAmB,CAAC;AAE3B,KAAK,UAAU,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEzE,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B,gGAAgG;IAChG,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC5B,0FAA0F;IAC1F,gBAAgB,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;IAChD;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B;AAED,gGAAgG;AAChG,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,mGAAmG;IACnG,MAAM,EAAE,OAAO,CAAC;IAChB,8EAA8E;IAC9E,eAAe,EAAE,OAAO,CAAC;IACzB,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,kBAAkB,EAAE,MAAM,CAAC;IAC3B,6DAA6D;IAC7D,aAAa,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACvC;AAED,6GAA6G;AAC7G,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,WAAW,CAAC;IACjB,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAC;IACd,6FAA6F;IAC7F,WAAW,EAAE,OAAO,CAAC;IACrB,mFAAmF;IACnF,YAAY,EAAE;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,GAAG,IAAI,CAAC;CACV;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,MAAM,CAAC;IAChC,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AA6HD,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAClC;;;;OAIG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,GAAG,SAAS,CAAC;IAC1C;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC,oEAAoE;IACpE,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACnC,6FAA6F;IAC7F,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,mGAAmG;IACnG,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B;;;;OAIG;IACH,SAAS,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;IACzC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;;;;;;OAWG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,GAAG,SAAS,CAAC;IACrD;;;;OAIG;IACH,WAAW,CAAC,EAAE,CAAC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IACrD;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IAClE;;;;;;;;;OASG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC;IAC9F;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IAC/F;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACrC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,aAAa,GAAG,SAAS,CAAC;IAC1C;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAChD;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GAAG,SAAS,CAAC;IAChF;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC;AAED,6FAA6F;AAC7F,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,cAGC;CACF;AAED;;;;;GAKG;AACH,qBAAa,cAAc;;IACzB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAa;IACnC,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC;kFAC8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAwB;IAC1D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAA2B;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC;yFACqF;IACrF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAqB;IAC1D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAqB;IACxD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA0C;IACtE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAyC;IACtE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAA0D;IACnF,4IAA4I;IAC5I,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsE;IAChG,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAEZ;IAGd,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,cAAc,CAE3B;IACJ,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,0EAA0E;IAC1E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA6C;IAI9E,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IACzC,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAU;IAChD;0GACsG;IACtG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAmB;IAChD,sGAAsG;IACtG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IAEvC,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgB;IAC3C,0FAA0F;IAC1F,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyC;IACnE,kFAAkF;IAClF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyC;IAChE,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAChE,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAGpC,oGAAoG;IACpG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAoB;IACjD,wEAAwE;IACxE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,sFAAsF;IACtF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAiC;IACjE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAyB;IACtD,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsC;IACtE,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsC;IACxE,yFAAyF;IACzF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyC;IACvE,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAsC;IACvE,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAqB;IAKrD,mFAAmF;IACnF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,iGAAiG;IACjG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAkC;IAC/D,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,0FAA0F;IAC1F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyC;IAClE;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA+B;IAC1D,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,IAAI,EAAE,kBAAkB,EA0InC;IAED;;;;OAIG;IACH,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAE7D;IAED;;;;;;OAMG;IACH;;;;OAIG;IACH,cAAc,CAAC,SAAS,EAAE,SAAS,iBAAiB,EAAE,GAAG,IAAI,CAE5D;IAED;;;OAGG;IACH,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAEtD;IAED;;;;OAIG;IACG,aAAa,CAAC,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAwBvE;IAED;;;;OAIG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAYvD;IAED;;;;OAIG;IACG,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAOxF;IAED,iBAAiB,CAAC,SAAS,EAAE,SAAS,iBAAiB,EAAE,GAAG,IAAI,CAY/D;IAED,uGAAuG;IACvG,IAAI,mBAAmB,IAAI,SAAS,iBAAiB,EAAE,CAEtD;IAED,uFAAuF;IACvF,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAgC;IAEvE,sFAAsF;IACtF,qBAAqB,CAAC,QAAQ,EAAE,SAAS,sBAAsB,EAAE,GAAG,IAAI,CAEvE;IAED,6FAA6F;IAC7F,IAAI,uBAAuB,IAAI,SAAS,sBAAsB,EAAE,CAE/D;IAED,wFAAwF;IACxF,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhF;IAED,yGAAyG;IACzG,cAAc,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAE7E;IAED;;;;OAIG;IACH,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,IAAI,CAGxD;IAED,mGAAmG;IACnG,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,QAAQ,CACN,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,EAAE,EAAE,UAAU,EACd,IAAI,CAAC,EAAE;QACL,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC5B,SAAS,CAAC,EAAE,eAAe,GAAG,SAAS,CAAC;QACxC,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;QAC/C,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC;QACvE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC/B,UAAU,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;QAC1C,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAChC,8FAA8F;QAC9F,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;KAC7B,GACA,IAAI,CAuBN;IAED;;;;;OAKG;IACH,cAAc,CACZ,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,SAAS,CAAC,EAAE,eAAe,CAAC;QAC5B,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QACnC,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACzD,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;KACrB,GACA,IAAI,CAmBN;IAED;;;;;;;;OAQG;YACW,wBAAwB;IAYtC;;;;OAIG;IACH,aAAa,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAEvC;IAED;oCACgC;IAChC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA2D;IAE7F;;;;;;;;;OASG;IACH,OAAO,CAAC,gBAAgB;IAcxB;;;;;;OAMG;IACH,cAAc,IAAI,IAAI,CAErB;IAED,sEAAsE;IACtE,SAAS,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,IAAI,CAG9C;IAED;;;;;OAKG;IACH,GAAG,CAAC,WAAW,EAAE,eAAe,GAAG,MAAM,IAAI,CAM5C;IAED;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IAarB;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,IAAI,CAGtD;IAED;;;;;;OAMG;IACH,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI,CAGvD;IAED,OAAO,CAAC,UAAU;IAUlB;0FACsF;IACtF,OAAO,CAAC,IAAI;IAYZ;wGACoG;IACpG,OAAO,CAAC,OAAO;IAUT,KAAK,CAAC,CAAC,SAAS,aAAa,EACjC,QAAQ,EAAE,CAAC,EACX,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,EACzB,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC,SAAS,CAAC,CAAC;IAChB,KAAK,CAAC,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC,SAAS,CAAC,CAAC;IAatB;;;;;;OAMG;YACW,SAAS;IAkGvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,UAAU;IAmClB,mGAAmG;IACnG,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAEjD;IAED;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAEhD;IAED;;;;;OAKG;IACH,cAAc,CACZ,IAAI,EAAE,YAAY,EAClB,KAAK,EAAE,aAAa,EACpB,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAK7B;IAED,yGAAyG;IACzG,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAExD;IAED;;;;;OAKG;IACH;;;;;;;OAOG;IACH,SAAS,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,GAAG,MAAM,IAAI,CAG9D;IAED;;;OAGG;IACG,YAAY,CAAC,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAuDxE;IAED;;;;;OAKG;IACH,OAAO,CACL,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAClF,MAAM,IAAI,CAGZ;IAED;;;;;;;;;;OAUG;IACG,cAAc,CAAC,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,MAAM,CAAC,CAgChE;IAEK,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAoB9C;IAEK,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAsC9C;IAED;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAevB,qEAAqE;IACrE,OAAO,CAAC,KAAK;IAMb;;;;;;OAMG;IACH,OAAO,CAAC,WAAW;IAMnB;;;;;;;;;;;;;;;;;;OAkBG;IACG,KAAK,CAAC,SAAS,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB7C;IAED;;;OAGG;YACW,oBAAoB;IAclC;;;;OAIG;IACG,aAAa,CAAC,GAAG,GAAE,MAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwD7D;IAED;;;OAGG;IACG,iBAAiB,CAAC,KAAK,GAAE,MAAqB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAmD1E;IAED;;;;OAIG;YACW,aAAa;IAwB3B;;;;;;OAMG;IACG,eAAe,CAAC,KAAK,GAAE,MAAqB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAKxE;IAED;;;;;OAKG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAGrD;IAED;;;;;;OAMG;YACW,cAAc;IAiB5B;;;;;;OAMG;IACG,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CA6E3E;IAED;;;;;;;;;OASG;IACG,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,SAAS,GAAG;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI,CAAC,CA8B7F;IAED;;;;OAIG;IACG,cAAc,CAClB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,OAAO,EACd,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAQnC;IAED;;;;;OAKG;IACG,UAAU,CAAC,KAAK,GAAE,MAAqB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAKnE;IAED;;;;;;;;;;OAUG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAoD/F;IAED;;;OAGG;YACW,YAAY;IAiC1B;;;OAGG;IACH;;;;;;;;;;;;;;;;;;OAkBG;IACG,YAAY,CAChB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,IAAI,CAAC,EAAE;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,GACvC,OAAO,CAAC,MAAM,CAAC,CAsDjB;IAED;;;;OAIG;YACW,aAAa;IAkB3B;;;;;;;;;;;OAWG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAmBvE;IAED;;;;;;;OAOG;IACG,eAAe,CACnB,QAAQ,EAAE,WAAW,EACrB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,EAC5C,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAM5B;IAED;;;;OAIG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAK1F;IAED,6FAA6F;IACvF,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAEpF;IAED;;;;OAIG;IACH,OAAO,CAAC,YAAY;IA6BpB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,kBAAkB;IAkB1B;;;;;OAKG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CA+DtF;IAED;;;;;;;;OAQG;IACG,WAAW,CACf,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,EACvC,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9B,OAAO,CAAC,SAAS,EAAE,CAAC,CAQtB;IAED;;;;;OAKG;YACW,cAAc;IAS5B;;;;;;;;;;;OAWG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAyB3D;IAED;;;;;;;OAOG;IACH,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEzD;IAED;;;;OAIG;IACG,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAOzE;IAED;;;yGAGqG;IACrG,WAAW,CAAC,KAAK,GAAE,MAAM,EAAO,GAAG,MAAM,EAAE,CAM1C;IAED;;;;gEAI4D;IACtD,YAAY,CAAC,KAAK,GAAE,MAAM,EAAO,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAM/D;IAED;;;;;;;;OAQG;YACW,gBAAgB;IAsD9B;;;;;;OAMG;YACW,SAAS;IA4BvB,oGAAoG;YACtF,YAAY;IA8B1B,oGAAoG;YACtF,QAAQ;YA8BR,OAAO;IA0DrB;;;;;OAKG;IACH;;;;;;OAMG;YACW,SAAS;IAmFvB;;;;;OAKG;YACW,iBAAiB;IAc/B;;;;;OAKG;IACH,OAAO,CAAC,eAAe;YAKT,kBAAkB;IA2FhC;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;YACW,eAAe;IA0E7B;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAKvB;yFACqF;IACrF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA6B;IAE7D;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;YACW,cAAc;IAS5B;uGACmG;YACrF,cAAc;IAmC5B,gGAAgG;YAClF,aAAa;IA0C3B;;;;;;yEAMqE;YACvD,aAAa;IAiP3B,6DAA6D;YAC/C,YAAY;IA+L1B;;;;;;;;;;;;;OAaG;YACW,eAAe;IAqE7B;;;;;;;;;;;;OAYG;YACW,oBAAoB;IAuElC;;;;;;OAMG;IACH,OAAO,CAAC,wBAAwB;IA6EhC;;wDAEoD;IACpD,OAAO,CAAC,UAAU;IAqDlB;;;;OAIG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAKvD;IAED;;;;OAIG;IACG,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAcxF;IAED;;;;OAIG;IACH,uBAAuB,CAAC,IAAI,EAC1B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,GAC9B,IAAI,CAEN;IAED;;;;;OAKG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,CAc7E;IAMD;;;0FAGsF;YACxE,kBAAkB;IAUhC;;;;;;OAMG;YACW,SAAS;IASvB;;;6EAGyE;YAC3D,cAAc;IAK5B;;;;;;;;OAQG;YACW,WAAW;IA2BzB;;qGAEiG;YACnF,cAAc;YAiBd,UAAU;IA8LxB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IA4B1B;;;;;;;;;;;;;;;;;;;;OAoBG;YACW,oBAAoB;IAclC;;;;;OAKG;YACW,sBAAsB;IA2EpC;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAQxB;;;;;;OAMG;YACW,gBAAgB;IAO9B,mGAAmG;YACrF,kBAAkB;IAwDhC,+FAA+F;YACjF,0BAA0B;IA0GxC,gGAAgG;YAClF,mBAAmB;IAgCjC;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;CAkD3B;AAED,8EAA8E;AAC9E,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAC3C,YAAY,SAAS,CAAC,EAAE,SAAS,EAIhC;CACF"}
@@ -1,6 +1,6 @@
1
1
  import { InMemoryAdmissionBackend } from './admission.js';
2
2
  import { backoffDelay, MAX_BACKOFF_MS } from './backoff.js';
3
- import { instantCheckpoint, stepCheckpoint } from './checkpoints.js';
3
+ import { instantCheckpoint, lostStepEvent, stepCheckpoint } from './checkpoints.js';
4
4
  import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispatch-routing.js';
5
5
  import { parseDuration } from './duration.js';
6
6
  import { Entities } from './entities.js';
@@ -305,9 +305,16 @@ export class WorkflowEngine {
305
305
  await this.completeRemoteResult(result);
306
306
  },
307
307
  // A heartbeat for an in-flight long step resets its liveness window (see callRemote) and is
308
- // persisted (throttled) on the step's checkpoint so liveness is visible cross-process.
308
+ // persisted (throttled) on the step's checkpoint so liveness is visible cross-process. With no
309
+ // in-memory waiter (the step suspended DURABLY, possibly on another instance) the beat rearms
310
+ // the step's durable LEASE instead, so a worker still holding a long step is never presumed
311
+ // lost and double-dispatched (see rearmStepLease).
309
312
  async (beat) => {
310
- this.heartbeatResets.get(beat.stepId)?.();
313
+ const reset = this.heartbeatResets.get(beat.stepId);
314
+ if (reset)
315
+ reset();
316
+ else
317
+ await this.rearmStepLease(beat.runId, beat.seq);
311
318
  this.persistHeartbeat(beat);
312
319
  },
313
320
  // A remote workflow worker streams each local step's lifecycle (running → completed/failed) so
@@ -2429,9 +2436,205 @@ export class WorkflowEngine {
2429
2436
  }
2430
2437
  // continue: persist any local steps the replay ran, dispatch the blocking ops, then suspend. When
2431
2438
  // those resolve (a result lands, a timer fires) `resume` brings us back for the next turn.
2432
- const wakeAt = await this.applyCommands(run, decision.commands);
2439
+ const wakeAt = await this.applyCommands(run, decision.commands, true);
2433
2440
  return this.settleRun(run, { kind: 'suspended', wakeAt });
2434
2441
  }
2442
+ /**
2443
+ * A polyglot workflow's re-emitted `call` whose checkpoint already exists: decide whether the step is
2444
+ * still out for delivery, or whether its dispatch was LOST and has to be re-driven. Returns the
2445
+ * deadline to park the run on (so the next decision lands exactly when it's due), or `undefined` when
2446
+ * there is nothing to wait for.
2447
+ *
2448
+ * WHY THIS EXISTS. A worker process killed mid-step (an OOM kill, exit 137 — a pod's memory ceiling,
2449
+ * not a bug in anyone's code) takes its in-flight job with it, and the transport cannot always put it
2450
+ * back: the `bullmq` transport's terminal-failure bridge (`worker.on('failed')`) only exists inside a
2451
+ * JS consumer, so a first-class Python fleet (see `docs/python.mdx`) has none; the `queue` transport's
2452
+ * reclaim sweep gives up past `maxStalledCount`, and never runs at all on an adapter without
2453
+ * `recoverStalledJobs`. NOTHING then settles the checkpoint. It stays `pending` — "out for delivery" —
2454
+ * and the `applyCommands` idempotency guard, which exists so a `ctx.gather_calls` fan-out doesn't
2455
+ * double-dispatch its still-in-flight siblings on every partial resume, skips it on every later turn.
2456
+ * The run wakes on the reconcile sweep, dispatches a turn, has the call re-emitted, skips it again,
2457
+ * and sleeps. Forever — `pending`, `attempts = 1`, `wakeAt` NULL.
2458
+ *
2459
+ * THE POLICY (mirrors {@link callRemote}'s `ctx.step` self-heal, which this path silently did not
2460
+ * share — so `remoteRedispatchMs` simply did not apply to a fan-out):
2461
+ * - a SETTLED checkpoint always wins: a step that completed just before the crash is never re-run;
2462
+ * - `remoteRedispatchMs` unset (the default) keeps the by-design "re-suspend, never re-dispatch";
2463
+ * - otherwise the pending checkpoint carries a LEASE in `wakeAt` — a result or a step heartbeat
2464
+ * ({@link rearmStepLease}) renews it, so a live worker still holding the step is never
2465
+ * double-dispatched — and only a LAPSED lease re-dispatches, bounded by `remoteRedispatchMax`;
2466
+ * - past the bound the step is failed `remote_step_lost` and the failure enters the run's history
2467
+ * (see {@link remoteHistory}), so the workflow's own error path surfaces it on the next turn
2468
+ * instead of the engine looping. The run is parked to wake immediately for exactly that turn.
2469
+ */
2470
+ async redriveLostCall(run, cp) {
2471
+ // Settled (or a non-remote seq): the checkpoint is the truth, nothing to re-drive.
2472
+ if (cp.kind !== 'remote' || cp.status !== 'pending')
2473
+ return undefined;
2474
+ if (this.remoteRedispatchMs == null)
2475
+ return undefined;
2476
+ // First sight of this pending step under the self-heal: stamp its lease (clock-space, persisted,
2477
+ // stable across replays and crashes) and park the run on it.
2478
+ if (cp.wakeAt == null) {
2479
+ const lease = this.clock() + this.remoteRedispatchMs;
2480
+ await this.store.saveCheckpoint({ ...cp, wakeAt: lease });
2481
+ return this.stepLeaseWakeAt(lease);
2482
+ }
2483
+ if (this.clock() < cp.wakeAt)
2484
+ return this.stepLeaseWakeAt(cp.wakeAt);
2485
+ // The lease lapsed with no result and no heartbeat: the dispatch is presumed lost.
2486
+ if (cp.attempts >= this.remoteRedispatchMax) {
2487
+ const error = {
2488
+ message: `remote step "${cp.name}" lost — no result after ${cp.attempts} dispatch(es)`,
2489
+ code: 'remote_step_lost',
2490
+ retryable: false,
2491
+ };
2492
+ const now = new Date();
2493
+ await this.store.saveCheckpoint({
2494
+ ...cp,
2495
+ status: 'failed',
2496
+ error,
2497
+ wakeAt: undefined,
2498
+ finishedAt: now,
2499
+ events: [...(cp.events ?? []), lostStepEvent(this.clock(), cp.attempts, 'lost')],
2500
+ });
2501
+ this.emit({
2502
+ type: 'step.failed',
2503
+ runId: run.id,
2504
+ seq: cp.seq,
2505
+ name: cp.name,
2506
+ kind: 'remote',
2507
+ error,
2508
+ });
2509
+ // Wake now: the failure is in history, so the very next turn's replay raises it inside the
2510
+ // gather and the run settles on the workflow's own error path.
2511
+ return this.clock();
2512
+ }
2513
+ const attempts = cp.attempts + 1;
2514
+ const lease = this.clock() + this.remoteRedispatchMs;
2515
+ const at = new Date();
2516
+ await this.store.saveCheckpoint({
2517
+ ...cp,
2518
+ attempts,
2519
+ wakeAt: lease,
2520
+ // The re-dispatch restarts the step's wall-clock window; the old one measured a worker that died.
2521
+ enqueuedAt: at,
2522
+ startedAt: at,
2523
+ finishedAt: at,
2524
+ // Observability: the step's own trail says WHY it ran again — a lost worker, not a failure retry.
2525
+ events: [...(cp.events ?? []), lostStepEvent(this.clock(), attempts, 'redispatched')],
2526
+ });
2527
+ await this.dispatchRemoteTask({
2528
+ runId: run.id,
2529
+ seq: cp.seq,
2530
+ name: cp.name,
2531
+ stepId: cp.stepId ?? stepId(run.id, cp.seq),
2532
+ group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cp.name), run.namespace),
2533
+ input: cp.input,
2534
+ attempt: attempts,
2535
+ });
2536
+ this.emit({
2537
+ type: 'step.started',
2538
+ runId: run.id,
2539
+ seq: cp.seq,
2540
+ name: cp.name,
2541
+ kind: 'remote',
2542
+ redispatched: true,
2543
+ });
2544
+ return this.stepLeaseWakeAt(lease);
2545
+ }
2546
+ /**
2547
+ * The run `wakeAt` a step lease should park on: the lease itself, but never LATER than the
2548
+ * {@link reconcileMs} safety net would have woken the run anyway. A long lease (a fan-out step that
2549
+ * legitimately runs for half an hour) must not stretch the orphan sweep's cadence for this run — an
2550
+ * early wake just re-parks on the un-lapsed lease, which is cheap and convergent.
2551
+ */
2552
+ stepLeaseWakeAt(lease) {
2553
+ const reconcile = this.reconcileMs;
2554
+ return reconcile == null ? lease : Math.min(lease, this.clock() + reconcile);
2555
+ }
2556
+ /** Earliest clock time a step's lease is worth LOOKING at again (`stepId` → clock ms) — the read
2557
+ * throttle for {@link rearmStepLease}, the sibling of {@link heartbeatPersists}. */
2558
+ stepLeaseRearms = new Map();
2559
+ /**
2560
+ * A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
2561
+ * that is still working on it is never presumed dead and double-dispatched. The durable counterpart
2562
+ * of the in-memory `heartbeatResets` rearm (which only exists on the instance that dispatched a
2563
+ * `timeoutMs` step). No-op unless the lost-dispatch self-heal is on — without `remoteRedispatchMs`
2564
+ * nothing ever re-dispatches, so there is no lease to defend.
2565
+ *
2566
+ * THROTTLED, for the same reason {@link persistHeartbeat} is: beats arrive every few seconds from a
2567
+ * hot batch loop, and defending a lease must not turn into a per-beat SELECT + UPDATE storm. The
2568
+ * throttle is the lease itself rather than a fixed interval — a renewal is only WORTH a write once
2569
+ * the window is half spent, because it buys at most one window either way. So:
2570
+ * - a beat that finds more than half the window still on the clock writes nothing, and records the
2571
+ * half-spent mark so subsequent beats skip even the `getCheckpoint` READ until then;
2572
+ * - a beat at or past the half-spent mark renews, leaving a full half-window of headroom — the
2573
+ * lease can never lapse under a worker whose beats are anywhere near their normal cadence.
2574
+ *
2575
+ * That bounds the cost at ONE read + ONE write per half `remoteRedispatchMs` per in-flight step
2576
+ * (with a 30-minute window: two writes an hour), independent of beat frequency. The in-memory mark
2577
+ * is a pure optimisation — cold on a fresh instance, or on the one that didn't dispatch the step —
2578
+ * and the durable half-spent check re-derives the same answer from the checkpoint, so nothing
2579
+ * depends on it surviving. Cleared wholesale past 1024 entries: a rare extra read, never a leak.
2580
+ *
2581
+ * NEVER REJECTS. This runs on the transport's SERIAL heartbeat handler, so a store hiccup thrown out
2582
+ * of here would take the whole fleet's beat loop with it — and every lease would then lapse at once,
2583
+ * a far worse failure than the write storm the throttle removes. So the entire rearm is best-effort,
2584
+ * exactly like {@link persistHeartbeat}: a missed renewal costs at most one re-drive of a step that
2585
+ * is idempotent by contract, and because the in-memory mark is advanced only AFTER a successful
2586
+ * write, the very next beat retries the renewal rather than waiting out a window.
2587
+ */
2588
+ async rearmStepLease(runId, seq) {
2589
+ if (this.remoteRedispatchMs == null)
2590
+ return;
2591
+ try {
2592
+ await this.renewStepLease(runId, seq);
2593
+ }
2594
+ catch {
2595
+ /* best-effort: a failed renewal must not break the transport's serial heartbeat loop */
2596
+ }
2597
+ }
2598
+ /** {@link rearmStepLease}'s body — every durable read/write of the lease renewal, so the caller's one
2599
+ * guard covers all of them (the read, the checkpoint write, and the run's wake carry-forward). */
2600
+ async renewStepLease(runId, seq) {
2601
+ if (this.remoteRedispatchMs == null)
2602
+ return;
2603
+ const id = stepId(runId, seq);
2604
+ const now = this.clock();
2605
+ const lookAgainAt = this.stepLeaseRearms.get(id);
2606
+ if (lookAgainAt !== undefined && now < lookAgainAt)
2607
+ return;
2608
+ const cp = await this.store.getCheckpoint(runId, seq);
2609
+ if (cp?.kind !== 'remote' || cp.status !== 'pending') {
2610
+ // Settled (or gone): stop tracking it, so a long-lived worker's stale beats don't keep the entry.
2611
+ this.stepLeaseRearms.delete(id);
2612
+ return;
2613
+ }
2614
+ const halfWindow = Math.ceil(this.remoteRedispatchMs / 2);
2615
+ // More than half the window left: nothing to buy, and nothing to look at until it is half spent.
2616
+ if (cp.wakeAt != null && cp.wakeAt > now + halfWindow) {
2617
+ this.stepLeaseRearms.set(id, cp.wakeAt - halfWindow);
2618
+ return;
2619
+ }
2620
+ const renewed = now + this.remoteRedispatchMs;
2621
+ const previous = cp.wakeAt;
2622
+ await this.store.saveCheckpoint({ ...cp, wakeAt: renewed });
2623
+ if (this.stepLeaseRearms.size > 1024)
2624
+ this.stepLeaseRearms.clear();
2625
+ this.stepLeaseRearms.set(id, renewed - halfWindow);
2626
+ // If the run is parked EXACTLY on this step's old lease, carry its wake forward too — otherwise a
2627
+ // beating worker costs one wasted turn per renewal window. Only on an exact match: any other
2628
+ // `wakeAt` belongs to some other timer (a sleep, a sibling's shorter lease) and moving it would
2629
+ // delay that instead. Failing HERE (after the checkpoint write landed) is harmless: the run wakes
2630
+ // on the old lease, finds it un-lapsed, and re-parks on the renewed one.
2631
+ if (previous == null)
2632
+ return;
2633
+ const run = await this.store.getRun(runId);
2634
+ if (run?.status === 'suspended' && run.wakeAt === previous) {
2635
+ await this.store.updateRun(runId, { wakeAt: renewed, updatedAt: new Date() });
2636
+ }
2637
+ }
2435
2638
  /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
2436
2639
  async remoteHistory(runId) {
2437
2640
  const checkpoints = await this.store.listCheckpoints(runId);
@@ -2475,8 +2678,13 @@ export class WorkflowEngine {
2475
2678
  return events.sort((a, b) => a.seq - b.seq);
2476
2679
  }
2477
2680
  /** Apply a turn's commands: persist recorded local steps, dispatch remote calls, schedule timers.
2478
- * Returns the earliest timer deadline to suspend on (or undefined — suspended on a result). */
2479
- async applyCommands(run, commands) {
2681
+ * Returns the earliest timer deadline to suspend on (or undefined — suspended on a result).
2682
+ *
2683
+ * `redriveLost` is set only by the `continue` path: a turn that PARKS on its blocking ops is the one
2684
+ * that may find a re-emitted `call` whose dispatch was lost and needs re-driving (see
2685
+ * {@link redriveLostCall}). A terminal turn (completed/failed/cancelled) applies its commands only to
2686
+ * record what already ran, and must never dispatch anything new. */
2687
+ async applyCommands(run, commands, redriveLost = false) {
2480
2688
  let wakeAt;
2481
2689
  for (const cmd of commands) {
2482
2690
  const at = new Date();
@@ -2519,13 +2727,29 @@ export class WorkflowEngine {
2519
2727
  // already exists — pending OR terminal — skip the save + dispatch entirely; its result lands
2520
2728
  // independently via the remote-result path (keyed by seq, so concurrent calls never clobber).
2521
2729
  // Mirrors the `startChild` `getRun(childId)` guard below.
2522
- if (await this.store.getCheckpoint(run.id, cmd.seq))
2730
+ //
2731
+ // What that guard must NOT do is treat "a checkpoint exists" as "somebody is still going to
2732
+ // deliver it". A dispatched job can be LOST — the worker was killed mid-step, so the job died
2733
+ // with the process and no result will ever come — and the checkpoint still reads `pending`.
2734
+ // Skipping such a command on every later turn is how a fan-out orphans its run (see
2735
+ // {@link redriveLostCall}, which owns that decision; a settled checkpoint always wins).
2736
+ const existing = await this.store.getCheckpoint(run.id, cmd.seq);
2737
+ if (existing) {
2738
+ const leaseAt = redriveLost ? await this.redriveLostCall(run, existing) : undefined;
2739
+ if (leaseAt != null)
2740
+ wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
2523
2741
  continue;
2742
+ }
2524
2743
  // Route this cross-SDK-worker `call` by the SAME name-based token a native `ctx.step`
2525
2744
  // dispatches with (and a worker subscribes to per handler name), so a decision-driven remote
2526
2745
  // step and an in-process one land on the identical queue. The command's own `group` is not a
2527
2746
  // routing input; the run's namespace is, so a tenant's fan-out stays on the tenant's pool.
2528
2747
  const callToken = tenantGroup(sanitizeQueueToken(cmd.name), run.namespace);
2748
+ // The dispatched step's LEASE: when the lost-dispatch self-heal is on (`remoteRedispatchMs`),
2749
+ // stamp the deadline by which a result — or a step heartbeat renewing it — must have landed,
2750
+ // and suspend the run ON it, so the re-drive lands when it's due instead of waiting for the
2751
+ // reconcile sweep. Unset (the default) leaves `wakeAt` NULL exactly as before.
2752
+ const lease = this.remoteRedispatchMs == null ? undefined : this.clock() + this.remoteRedispatchMs;
2529
2753
  await this.store.saveCheckpoint(stepCheckpoint({
2530
2754
  runId: run.id,
2531
2755
  seq: cmd.seq,
@@ -2534,6 +2758,7 @@ export class WorkflowEngine {
2534
2758
  status: 'pending',
2535
2759
  input: cmd.input,
2536
2760
  attempts: 1,
2761
+ wakeAt: lease,
2537
2762
  workerGroup: callToken,
2538
2763
  enqueuedAt: at,
2539
2764
  startedAt: at,
@@ -2561,6 +2786,10 @@ export class WorkflowEngine {
2561
2786
  name: cmd.name,
2562
2787
  kind: 'remote',
2563
2788
  });
2789
+ if (lease != null) {
2790
+ const leaseWake = this.stepLeaseWakeAt(lease);
2791
+ wakeAt = wakeAt == null ? leaseWake : Math.min(wakeAt, leaseWake);
2792
+ }
2564
2793
  }
2565
2794
  else if (cmd.kind === 'sleep') {
2566
2795
  const deadline = this.clock() + cmd.ms;
@@ -3350,6 +3579,12 @@ export class WorkflowEngine {
3350
3579
  enqueuedAt: reEnqueuedAt,
3351
3580
  startedAt: reEnqueuedAt,
3352
3581
  finishedAt: reEnqueuedAt,
3582
+ // Same observable marker the polyglot `call` path stamps (see redriveLostCall): the step's own
3583
+ // trail distinguishes a lost-dispatch re-drive from a failure retry.
3584
+ events: [
3585
+ ...(existing.events ?? []),
3586
+ lostStepEvent(this.clock(), reAttempt, 'redispatched'),
3587
+ ],
3353
3588
  });
3354
3589
  await this.dispatchRemoteTask({
3355
3590
  runId,
@@ -3362,6 +3597,14 @@ export class WorkflowEngine {
3362
3597
  attempt: reAttempt,
3363
3598
  transport,
3364
3599
  });
3600
+ this.emit({
3601
+ type: 'step.started',
3602
+ runId,
3603
+ seq,
3604
+ name: step.name,
3605
+ kind: 'remote',
3606
+ redispatched: true,
3607
+ });
3365
3608
  throw new WorkflowSuspended(nextDeadline, [seq]);
3366
3609
  }
3367
3610
  // Durable retry: a failed attempt re-dispatches up to `retries`, spacing attempts by `backoff` —