@dudousxd/nestjs-durable-core 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +136 -48
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +51 -7
- package/dist/index.d.ts +51 -7
- package/dist/index.js +136 -48
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -209,7 +209,15 @@ declare function workflowName(ref: WorkflowRef): string;
|
|
|
209
209
|
*/
|
|
210
210
|
type RunStatus =
|
|
211
211
|
/** Created + enqueued by `start`, not yet picked up — a worker will lease and execute it. */
|
|
212
|
-
'pending' | 'running' | 'suspended'
|
|
212
|
+
'pending' | 'running' | 'suspended'
|
|
213
|
+
/**
|
|
214
|
+
* Compensating cancel in progress: `cancel(runId, { compensate: true })` has been requested and the
|
|
215
|
+
* saga undo is running in the background. NON-TERMINAL — the run is still in-flight (counted by the
|
|
216
|
+
* admission gate, re-driven by recovery if the engine crashes mid-compensation), and flips to
|
|
217
|
+
* `cancelled` once compensation completes. A non-compensating cancel skips this and goes straight to
|
|
218
|
+
* `cancelled`.
|
|
219
|
+
*/
|
|
220
|
+
| 'cancelling' | 'completed' | 'failed' | 'cancelled'
|
|
213
221
|
/** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill). Terminal; inspect it. */
|
|
214
222
|
| 'dead';
|
|
215
223
|
/** One execution of a workflow. The unit of durability and the unit shown in the dashboard. */
|
|
@@ -577,7 +585,13 @@ interface StepResult {
|
|
|
577
585
|
interface Heartbeat {
|
|
578
586
|
runId: string;
|
|
579
587
|
seq: number;
|
|
580
|
-
|
|
588
|
+
/**
|
|
589
|
+
* The in-flight step this beat keeps alive. ABSENT for a RUN-scoped (workflow-turn) heartbeat: a
|
|
590
|
+
* worker replaying a long workflow turn beats for its `runId` so the engine rearms the run's
|
|
591
|
+
* `advance` deadline (see `WorkflowEngineDeps.remoteAdvanceSilenceMs`). The engine then keys the
|
|
592
|
+
* liveness reset by `runId` instead of `stepId`.
|
|
593
|
+
*/
|
|
594
|
+
stepId?: string | undefined;
|
|
581
595
|
group: string;
|
|
582
596
|
}
|
|
583
597
|
/** engine → workflow worker: advance this run one turn by replaying the function against `history`. */
|
|
@@ -1291,6 +1305,18 @@ interface WorkflowEngineDeps {
|
|
|
1291
1305
|
* Omit for unlimited (the default — recovery always retries).
|
|
1292
1306
|
*/
|
|
1293
1307
|
maxRecoveryAttempts?: number | undefined;
|
|
1308
|
+
/**
|
|
1309
|
+
* Opt-in liveness deadline (ms) for a remote workflow `advance`. If the worker neither returns a
|
|
1310
|
+
* decision NOR sends a run-scoped {@link Heartbeat} within this window, the engine presumes it dead:
|
|
1311
|
+
* it rejects the advance with {@link RemoteWorkflowTimeout}, releases the lease, and lets recovery
|
|
1312
|
+
* re-drive (replaying completed steps from history). Each beat REARMS the window, so a worker still
|
|
1313
|
+
* legitimately replaying a long turn (it keeps beating) is never re-driven — closing the duplicate-
|
|
1314
|
+
* side-effect hazard a fixed {@link RemoteWorkflowExecutor} `timeoutMs` cannot avoid. Omit (default)
|
|
1315
|
+
* for the prior unbounded await — byte-identical behavior when unset. If a `RemoteWorkflowExecutor` is
|
|
1316
|
+
* ALSO constructed with its own `timeoutMs`, the shorter of the two deadlines wins; prefer setting only
|
|
1317
|
+
* one (this rearmed window for heartbeat-aware workers, or the fixed `timeoutMs` otherwise).
|
|
1318
|
+
*/
|
|
1319
|
+
remoteAdvanceSilenceMs?: number | undefined;
|
|
1294
1320
|
/**
|
|
1295
1321
|
* Build the public callback URL for a `ctx.webhook()` token (e.g.
|
|
1296
1322
|
* ``(t) => `https://api.example.com/durable/webhooks/${t}` ``). Populates
|
|
@@ -1364,6 +1390,7 @@ declare class WorkflowEngine {
|
|
|
1364
1390
|
private readonly instanceId;
|
|
1365
1391
|
private readonly leaseMs;
|
|
1366
1392
|
private readonly maxRecoveryAttempts?;
|
|
1393
|
+
private readonly remoteAdvanceSilenceMs?;
|
|
1367
1394
|
private readonly webhookUrl?;
|
|
1368
1395
|
private readonly traceparent?;
|
|
1369
1396
|
private readonly context?;
|
|
@@ -1388,7 +1415,9 @@ declare class WorkflowEngine {
|
|
|
1388
1415
|
private readonly accumulators;
|
|
1389
1416
|
/** In-flight remote steps awaiting a worker result, keyed by stepId. */
|
|
1390
1417
|
private readonly pending;
|
|
1391
|
-
/**
|
|
1418
|
+
/** "Reset the liveness timer" callbacks keyed by the beat's target — a stepId (in-flight remote step)
|
|
1419
|
+
* or a runId (in-flight remote workflow turn) — invoked when a matching heartbeat arrives. The two id
|
|
1420
|
+
* shapes don't collide in practice: a stepId is `${runId}:${seq}`, a run key is the bare runId. */
|
|
1392
1421
|
private readonly heartbeatResets;
|
|
1393
1422
|
private readonly listeners;
|
|
1394
1423
|
/** Step interceptors (onion middleware around real local-step execution), first = outermost. */
|
|
@@ -1730,6 +1759,13 @@ declare class WorkflowEngine {
|
|
|
1730
1759
|
/** Apply a turn's commands: persist recorded local steps, dispatch remote calls, schedule timers.
|
|
1731
1760
|
* Returns the earliest timer deadline to suspend on (or undefined — suspended on a result). */
|
|
1732
1761
|
private applyCommands;
|
|
1762
|
+
/**
|
|
1763
|
+
* Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
|
|
1764
|
+
* exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
|
|
1765
|
+
* suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
|
|
1766
|
+
* stop at a suspension point. Clears the in-memory cancel flag.
|
|
1767
|
+
*/
|
|
1768
|
+
private compensateAndCancel;
|
|
1733
1769
|
/** The run body, lease held + renewed by {@link execute}. */
|
|
1734
1770
|
private runExecution;
|
|
1735
1771
|
/**
|
|
@@ -1785,10 +1821,15 @@ declare class WorkflowEngine {
|
|
|
1785
1821
|
/** In-memory await path for a remote step with a liveness `timeoutMs` (re-dispatch on timeout). */
|
|
1786
1822
|
private callRemoteInMemory;
|
|
1787
1823
|
/**
|
|
1788
|
-
* Await
|
|
1789
|
-
* arrives within `timeoutMs`. Each heartbeat (delivered via `transport.onHeartbeat
|
|
1790
|
-
* window, so a worker that keeps beating stays alive past
|
|
1824
|
+
* Await `resultPromise`, but reject with `onTimeout()` if neither it settles nor a heartbeat for `id`
|
|
1825
|
+
* arrives within `timeoutMs`. Each heartbeat (delivered via `transport.onHeartbeat`, looked up in
|
|
1826
|
+
* `heartbeatResets` by `id`) REARMS the window, so a worker that keeps beating stays alive past
|
|
1827
|
+
* `timeoutMs`. Shared by the remote-STEP liveness (keyed by stepId) and the remote-WORKFLOW advance
|
|
1828
|
+
* liveness (keyed by runId); the caller supplies the timeout error and any extra cleanup via `onTimeout`.
|
|
1791
1829
|
*/
|
|
1830
|
+
private awaitWithLivenessDeadline;
|
|
1831
|
+
/** Remote-STEP liveness: await a worker result, timing out (with the step's pending-map cleanup) to
|
|
1832
|
+
* `RemoteStepTimeout` if it neither lands nor beats within `timeoutMs`. See {@link awaitWithLivenessDeadline}. */
|
|
1792
1833
|
private awaitWithHeartbeat;
|
|
1793
1834
|
}
|
|
1794
1835
|
/** Raised inside the workflow when a remote worker reports a step failure. */
|
|
@@ -2301,13 +2342,16 @@ declare class InMemoryStateStore implements StateStore {
|
|
|
2301
2342
|
declare class InMemoryTransport implements Transport, ControlPlane {
|
|
2302
2343
|
private readonly handlers;
|
|
2303
2344
|
private resultHandler?;
|
|
2345
|
+
private heartbeatHandler?;
|
|
2304
2346
|
private stepEventHandler?;
|
|
2305
2347
|
private readonly controlHandlers;
|
|
2306
2348
|
/** Register a fake worker handler for a step name. */
|
|
2307
2349
|
handle(name: string, fn: StepHandler): void;
|
|
2308
2350
|
dispatch(task: RemoteTask): Promise<void>;
|
|
2309
2351
|
onResult(handler: (result: StepResult) => Promise<void>): void;
|
|
2310
|
-
onHeartbeat(
|
|
2352
|
+
onHeartbeat(handler: (beat: Heartbeat) => Promise<void>): void;
|
|
2353
|
+
/** Test hook: deliver a liveness heartbeat to the engine, as a real broker would on its channel. */
|
|
2354
|
+
emitHeartbeat(beat: Heartbeat): Promise<void>;
|
|
2311
2355
|
/** Workflow step lifecycle: deliver to the engine's handler (async, mirroring a real broker). */
|
|
2312
2356
|
dispatchStepEvent(event: WorkflowStepEvent): Promise<void>;
|
|
2313
2357
|
onStepEvent(handler: (event: WorkflowStepEvent) => Promise<void>): void;
|
package/dist/index.d.ts
CHANGED
|
@@ -209,7 +209,15 @@ declare function workflowName(ref: WorkflowRef): string;
|
|
|
209
209
|
*/
|
|
210
210
|
type RunStatus =
|
|
211
211
|
/** Created + enqueued by `start`, not yet picked up — a worker will lease and execute it. */
|
|
212
|
-
'pending' | 'running' | 'suspended'
|
|
212
|
+
'pending' | 'running' | 'suspended'
|
|
213
|
+
/**
|
|
214
|
+
* Compensating cancel in progress: `cancel(runId, { compensate: true })` has been requested and the
|
|
215
|
+
* saga undo is running in the background. NON-TERMINAL — the run is still in-flight (counted by the
|
|
216
|
+
* admission gate, re-driven by recovery if the engine crashes mid-compensation), and flips to
|
|
217
|
+
* `cancelled` once compensation completes. A non-compensating cancel skips this and goes straight to
|
|
218
|
+
* `cancelled`.
|
|
219
|
+
*/
|
|
220
|
+
| 'cancelling' | 'completed' | 'failed' | 'cancelled'
|
|
213
221
|
/** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill). Terminal; inspect it. */
|
|
214
222
|
| 'dead';
|
|
215
223
|
/** One execution of a workflow. The unit of durability and the unit shown in the dashboard. */
|
|
@@ -577,7 +585,13 @@ interface StepResult {
|
|
|
577
585
|
interface Heartbeat {
|
|
578
586
|
runId: string;
|
|
579
587
|
seq: number;
|
|
580
|
-
|
|
588
|
+
/**
|
|
589
|
+
* The in-flight step this beat keeps alive. ABSENT for a RUN-scoped (workflow-turn) heartbeat: a
|
|
590
|
+
* worker replaying a long workflow turn beats for its `runId` so the engine rearms the run's
|
|
591
|
+
* `advance` deadline (see `WorkflowEngineDeps.remoteAdvanceSilenceMs`). The engine then keys the
|
|
592
|
+
* liveness reset by `runId` instead of `stepId`.
|
|
593
|
+
*/
|
|
594
|
+
stepId?: string | undefined;
|
|
581
595
|
group: string;
|
|
582
596
|
}
|
|
583
597
|
/** engine → workflow worker: advance this run one turn by replaying the function against `history`. */
|
|
@@ -1291,6 +1305,18 @@ interface WorkflowEngineDeps {
|
|
|
1291
1305
|
* Omit for unlimited (the default — recovery always retries).
|
|
1292
1306
|
*/
|
|
1293
1307
|
maxRecoveryAttempts?: number | undefined;
|
|
1308
|
+
/**
|
|
1309
|
+
* Opt-in liveness deadline (ms) for a remote workflow `advance`. If the worker neither returns a
|
|
1310
|
+
* decision NOR sends a run-scoped {@link Heartbeat} within this window, the engine presumes it dead:
|
|
1311
|
+
* it rejects the advance with {@link RemoteWorkflowTimeout}, releases the lease, and lets recovery
|
|
1312
|
+
* re-drive (replaying completed steps from history). Each beat REARMS the window, so a worker still
|
|
1313
|
+
* legitimately replaying a long turn (it keeps beating) is never re-driven — closing the duplicate-
|
|
1314
|
+
* side-effect hazard a fixed {@link RemoteWorkflowExecutor} `timeoutMs` cannot avoid. Omit (default)
|
|
1315
|
+
* for the prior unbounded await — byte-identical behavior when unset. If a `RemoteWorkflowExecutor` is
|
|
1316
|
+
* ALSO constructed with its own `timeoutMs`, the shorter of the two deadlines wins; prefer setting only
|
|
1317
|
+
* one (this rearmed window for heartbeat-aware workers, or the fixed `timeoutMs` otherwise).
|
|
1318
|
+
*/
|
|
1319
|
+
remoteAdvanceSilenceMs?: number | undefined;
|
|
1294
1320
|
/**
|
|
1295
1321
|
* Build the public callback URL for a `ctx.webhook()` token (e.g.
|
|
1296
1322
|
* ``(t) => `https://api.example.com/durable/webhooks/${t}` ``). Populates
|
|
@@ -1364,6 +1390,7 @@ declare class WorkflowEngine {
|
|
|
1364
1390
|
private readonly instanceId;
|
|
1365
1391
|
private readonly leaseMs;
|
|
1366
1392
|
private readonly maxRecoveryAttempts?;
|
|
1393
|
+
private readonly remoteAdvanceSilenceMs?;
|
|
1367
1394
|
private readonly webhookUrl?;
|
|
1368
1395
|
private readonly traceparent?;
|
|
1369
1396
|
private readonly context?;
|
|
@@ -1388,7 +1415,9 @@ declare class WorkflowEngine {
|
|
|
1388
1415
|
private readonly accumulators;
|
|
1389
1416
|
/** In-flight remote steps awaiting a worker result, keyed by stepId. */
|
|
1390
1417
|
private readonly pending;
|
|
1391
|
-
/**
|
|
1418
|
+
/** "Reset the liveness timer" callbacks keyed by the beat's target — a stepId (in-flight remote step)
|
|
1419
|
+
* or a runId (in-flight remote workflow turn) — invoked when a matching heartbeat arrives. The two id
|
|
1420
|
+
* shapes don't collide in practice: a stepId is `${runId}:${seq}`, a run key is the bare runId. */
|
|
1392
1421
|
private readonly heartbeatResets;
|
|
1393
1422
|
private readonly listeners;
|
|
1394
1423
|
/** Step interceptors (onion middleware around real local-step execution), first = outermost. */
|
|
@@ -1730,6 +1759,13 @@ declare class WorkflowEngine {
|
|
|
1730
1759
|
/** Apply a turn's commands: persist recorded local steps, dispatch remote calls, schedule timers.
|
|
1731
1760
|
* Returns the earliest timer deadline to suspend on (or undefined — suspended on a result). */
|
|
1732
1761
|
private applyCommands;
|
|
1762
|
+
/**
|
|
1763
|
+
* Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
|
|
1764
|
+
* exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
|
|
1765
|
+
* suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
|
|
1766
|
+
* stop at a suspension point. Clears the in-memory cancel flag.
|
|
1767
|
+
*/
|
|
1768
|
+
private compensateAndCancel;
|
|
1733
1769
|
/** The run body, lease held + renewed by {@link execute}. */
|
|
1734
1770
|
private runExecution;
|
|
1735
1771
|
/**
|
|
@@ -1785,10 +1821,15 @@ declare class WorkflowEngine {
|
|
|
1785
1821
|
/** In-memory await path for a remote step with a liveness `timeoutMs` (re-dispatch on timeout). */
|
|
1786
1822
|
private callRemoteInMemory;
|
|
1787
1823
|
/**
|
|
1788
|
-
* Await
|
|
1789
|
-
* arrives within `timeoutMs`. Each heartbeat (delivered via `transport.onHeartbeat
|
|
1790
|
-
* window, so a worker that keeps beating stays alive past
|
|
1824
|
+
* Await `resultPromise`, but reject with `onTimeout()` if neither it settles nor a heartbeat for `id`
|
|
1825
|
+
* arrives within `timeoutMs`. Each heartbeat (delivered via `transport.onHeartbeat`, looked up in
|
|
1826
|
+
* `heartbeatResets` by `id`) REARMS the window, so a worker that keeps beating stays alive past
|
|
1827
|
+
* `timeoutMs`. Shared by the remote-STEP liveness (keyed by stepId) and the remote-WORKFLOW advance
|
|
1828
|
+
* liveness (keyed by runId); the caller supplies the timeout error and any extra cleanup via `onTimeout`.
|
|
1791
1829
|
*/
|
|
1830
|
+
private awaitWithLivenessDeadline;
|
|
1831
|
+
/** Remote-STEP liveness: await a worker result, timing out (with the step's pending-map cleanup) to
|
|
1832
|
+
* `RemoteStepTimeout` if it neither lands nor beats within `timeoutMs`. See {@link awaitWithLivenessDeadline}. */
|
|
1792
1833
|
private awaitWithHeartbeat;
|
|
1793
1834
|
}
|
|
1794
1835
|
/** Raised inside the workflow when a remote worker reports a step failure. */
|
|
@@ -2301,13 +2342,16 @@ declare class InMemoryStateStore implements StateStore {
|
|
|
2301
2342
|
declare class InMemoryTransport implements Transport, ControlPlane {
|
|
2302
2343
|
private readonly handlers;
|
|
2303
2344
|
private resultHandler?;
|
|
2345
|
+
private heartbeatHandler?;
|
|
2304
2346
|
private stepEventHandler?;
|
|
2305
2347
|
private readonly controlHandlers;
|
|
2306
2348
|
/** Register a fake worker handler for a step name. */
|
|
2307
2349
|
handle(name: string, fn: StepHandler): void;
|
|
2308
2350
|
dispatch(task: RemoteTask): Promise<void>;
|
|
2309
2351
|
onResult(handler: (result: StepResult) => Promise<void>): void;
|
|
2310
|
-
onHeartbeat(
|
|
2352
|
+
onHeartbeat(handler: (beat: Heartbeat) => Promise<void>): void;
|
|
2353
|
+
/** Test hook: deliver a liveness heartbeat to the engine, as a real broker would on its channel. */
|
|
2354
|
+
emitHeartbeat(beat: Heartbeat): Promise<void>;
|
|
2311
2355
|
/** Workflow step lifecycle: deliver to the engine's handler (async, mirroring a real broker). */
|
|
2312
2356
|
dispatchStepEvent(event: WorkflowStepEvent): Promise<void>;
|
|
2313
2357
|
onStepEvent(handler: (event: WorkflowStepEvent) => Promise<void>): void;
|
package/dist/index.js
CHANGED
|
@@ -733,7 +733,8 @@ var SingletonGate = class {
|
|
|
733
733
|
statuses: [
|
|
734
734
|
"pending",
|
|
735
735
|
"running",
|
|
736
|
-
"suspended"
|
|
736
|
+
"suspended",
|
|
737
|
+
"cancelling"
|
|
737
738
|
]
|
|
738
739
|
});
|
|
739
740
|
if (queued.length >= cap) {
|
|
@@ -751,7 +752,8 @@ var SingletonGate = class {
|
|
|
751
752
|
workflow: run.workflow,
|
|
752
753
|
statuses: [
|
|
753
754
|
"running",
|
|
754
|
-
"suspended"
|
|
755
|
+
"suspended",
|
|
756
|
+
"cancelling"
|
|
755
757
|
]
|
|
756
758
|
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
757
759
|
const idx = inflight.findIndex((r) => r.id === run.id);
|
|
@@ -1431,6 +1433,7 @@ var WorkflowEngine = class {
|
|
|
1431
1433
|
instanceId;
|
|
1432
1434
|
leaseMs;
|
|
1433
1435
|
maxRecoveryAttempts;
|
|
1436
|
+
remoteAdvanceSilenceMs;
|
|
1434
1437
|
webhookUrl;
|
|
1435
1438
|
traceparent;
|
|
1436
1439
|
context;
|
|
@@ -1455,7 +1458,9 @@ var WorkflowEngine = class {
|
|
|
1455
1458
|
accumulators;
|
|
1456
1459
|
/** In-flight remote steps awaiting a worker result, keyed by stepId. */
|
|
1457
1460
|
pending = /* @__PURE__ */ new Map();
|
|
1458
|
-
/**
|
|
1461
|
+
/** "Reset the liveness timer" callbacks keyed by the beat's target — a stepId (in-flight remote step)
|
|
1462
|
+
* or a runId (in-flight remote workflow turn) — invoked when a matching heartbeat arrives. The two id
|
|
1463
|
+
* shapes don't collide in practice: a stepId is `${runId}:${seq}`, a run key is the bare runId. */
|
|
1459
1464
|
heartbeatResets = /* @__PURE__ */ new Map();
|
|
1460
1465
|
listeners = /* @__PURE__ */ new Set();
|
|
1461
1466
|
/** Step interceptors (onion middleware around real local-step execution), first = outermost. */
|
|
@@ -1494,6 +1499,7 @@ var WorkflowEngine = class {
|
|
|
1494
1499
|
this.instanceId = deps.instanceId ?? globalThis.crypto.randomUUID();
|
|
1495
1500
|
this.leaseMs = deps.leaseMs ?? 3e4;
|
|
1496
1501
|
this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
|
|
1502
|
+
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
1497
1503
|
this.webhookUrl = deps.webhookUrl;
|
|
1498
1504
|
this.traceparent = deps.traceparent;
|
|
1499
1505
|
this.context = deps.context;
|
|
@@ -1530,9 +1536,11 @@ var WorkflowEngine = class {
|
|
|
1530
1536
|
}
|
|
1531
1537
|
await this.completeRemoteResult(result);
|
|
1532
1538
|
},
|
|
1533
|
-
// A heartbeat for an in-flight long
|
|
1539
|
+
// A heartbeat resets the liveness window for whatever it targets: an in-flight long STEP (keyed
|
|
1540
|
+
// by stepId, see callRemote) or — when stepId is absent — the in-flight workflow TURN (keyed by
|
|
1541
|
+
// runId, see runRemoteExecution's heartbeat-rearmed advance).
|
|
1534
1542
|
async (beat) => {
|
|
1535
|
-
this.heartbeatResets.get(beat.stepId)?.();
|
|
1543
|
+
this.heartbeatResets.get(beat.stepId ?? beat.runId)?.();
|
|
1536
1544
|
},
|
|
1537
1545
|
// A remote workflow worker streams each local step's lifecycle (running → completed/failed) so
|
|
1538
1546
|
// it's checkpointed live, not all-at-once when the long turn ends.
|
|
@@ -1894,14 +1902,16 @@ var WorkflowEngine = class {
|
|
|
1894
1902
|
continue;
|
|
1895
1903
|
}
|
|
1896
1904
|
await this.store.releaseRunLock(run.id);
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1905
|
+
if (run.status !== "cancelling") {
|
|
1906
|
+
await this.store.updateRun(run.id, {
|
|
1907
|
+
status: "pending",
|
|
1908
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1909
|
+
});
|
|
1910
|
+
}
|
|
1901
1911
|
await this.runDispatcher.dispatch(run.id);
|
|
1902
1912
|
results.push({
|
|
1903
1913
|
runId: run.id,
|
|
1904
|
-
status: "pending"
|
|
1914
|
+
status: run.status === "cancelling" ? "cancelling" : "pending"
|
|
1905
1915
|
});
|
|
1906
1916
|
}
|
|
1907
1917
|
return results;
|
|
@@ -2020,7 +2030,7 @@ var WorkflowEngine = class {
|
|
|
2020
2030
|
* `start` when a call site needs the outcome — `await start(...); const r = await waitForRun(id)`.
|
|
2021
2031
|
*/
|
|
2022
2032
|
waitForRun(runId, opts) {
|
|
2023
|
-
const isSettled = /* @__PURE__ */ __name((s) => s !== "pending" && s !== "running", "isSettled");
|
|
2033
|
+
const isSettled = /* @__PURE__ */ __name((s) => s !== "pending" && s !== "running" && s !== "cancelling", "isSettled");
|
|
2024
2034
|
const toResult = /* @__PURE__ */ __name((run) => ({
|
|
2025
2035
|
runId,
|
|
2026
2036
|
status: run.status,
|
|
@@ -2184,8 +2194,19 @@ var WorkflowEngine = class {
|
|
|
2184
2194
|
error: run.error
|
|
2185
2195
|
};
|
|
2186
2196
|
}
|
|
2197
|
+
if (run.status === "cancelling") {
|
|
2198
|
+
return {
|
|
2199
|
+
runId,
|
|
2200
|
+
status: "cancelling"
|
|
2201
|
+
};
|
|
2202
|
+
}
|
|
2187
2203
|
if (opts?.compensate && (run.status === "suspended" || run.status === "running")) {
|
|
2188
2204
|
this.cancelRequested.add(runId);
|
|
2205
|
+
await this.store.updateRun(runId, {
|
|
2206
|
+
status: "cancelling",
|
|
2207
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2208
|
+
});
|
|
2209
|
+
run.status = "cancelling";
|
|
2189
2210
|
if (this.controlPlane) {
|
|
2190
2211
|
void this.controlPlane.publishControl({
|
|
2191
2212
|
kind: "cancel",
|
|
@@ -2199,7 +2220,7 @@ var WorkflowEngine = class {
|
|
|
2199
2220
|
await this.cancelChildren(runId, opts);
|
|
2200
2221
|
return {
|
|
2201
2222
|
runId,
|
|
2202
|
-
status:
|
|
2223
|
+
status: "cancelling"
|
|
2203
2224
|
};
|
|
2204
2225
|
}
|
|
2205
2226
|
const error = {
|
|
@@ -2590,6 +2611,36 @@ var WorkflowEngine = class {
|
|
|
2590
2611
|
};
|
|
2591
2612
|
}
|
|
2592
2613
|
async runRemoteExecution(run, registered) {
|
|
2614
|
+
if (run.status === "cancelling") {
|
|
2615
|
+
const error = {
|
|
2616
|
+
message: "cancelled"
|
|
2617
|
+
};
|
|
2618
|
+
await this.store.updateRun(run.id, {
|
|
2619
|
+
status: "cancelled",
|
|
2620
|
+
error,
|
|
2621
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2622
|
+
});
|
|
2623
|
+
this.emit({
|
|
2624
|
+
type: "run.failed",
|
|
2625
|
+
runId: run.id,
|
|
2626
|
+
workflow: run.workflow,
|
|
2627
|
+
error
|
|
2628
|
+
});
|
|
2629
|
+
this.notifyCancelled(run.id);
|
|
2630
|
+
if (this.controlPlane) {
|
|
2631
|
+
void this.controlPlane.publishControl({
|
|
2632
|
+
kind: "cancel",
|
|
2633
|
+
runId: run.id,
|
|
2634
|
+
from: this.instanceId
|
|
2635
|
+
}).catch(() => void 0);
|
|
2636
|
+
}
|
|
2637
|
+
await this.store.releaseRunLock(run.id);
|
|
2638
|
+
return {
|
|
2639
|
+
runId: run.id,
|
|
2640
|
+
status: "cancelled",
|
|
2641
|
+
error
|
|
2642
|
+
};
|
|
2643
|
+
}
|
|
2593
2644
|
const remote = registered.remote;
|
|
2594
2645
|
if (run.status === "pending") {
|
|
2595
2646
|
await this.store.updateRun(run.id, {
|
|
@@ -2624,7 +2675,9 @@ var WorkflowEngine = class {
|
|
|
2624
2675
|
const history = await this.remoteHistory(run.id);
|
|
2625
2676
|
let decision;
|
|
2626
2677
|
try {
|
|
2627
|
-
|
|
2678
|
+
const silenceMs = this.remoteAdvanceSilenceMs;
|
|
2679
|
+
const advance = remote.executor.advance(run, history);
|
|
2680
|
+
decision = silenceMs != null ? await this.awaitWithLivenessDeadline(run.id, advance, silenceMs, () => new RemoteWorkflowTimeout(run.id, silenceMs)) : await advance;
|
|
2628
2681
|
} catch (err) {
|
|
2629
2682
|
if (err instanceof RemoteWorkflowTimeout) {
|
|
2630
2683
|
await this.store.releaseRunLock(run.id);
|
|
@@ -2681,6 +2734,15 @@ var WorkflowEngine = class {
|
|
|
2681
2734
|
status: "cancelled"
|
|
2682
2735
|
};
|
|
2683
2736
|
}
|
|
2737
|
+
const fresh = await this.store.getRun(run.id);
|
|
2738
|
+
if (fresh && (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead")) {
|
|
2739
|
+
return {
|
|
2740
|
+
runId: run.id,
|
|
2741
|
+
status: fresh.status,
|
|
2742
|
+
output: fresh.output,
|
|
2743
|
+
error: fresh.error
|
|
2744
|
+
};
|
|
2745
|
+
}
|
|
2684
2746
|
const wakeAt = await this.applyCommands(run, decision.commands);
|
|
2685
2747
|
return this.settleRun(run, {
|
|
2686
2748
|
kind: "suspended",
|
|
@@ -2848,8 +2910,41 @@ var WorkflowEngine = class {
|
|
|
2848
2910
|
}
|
|
2849
2911
|
return wakeAt;
|
|
2850
2912
|
}
|
|
2913
|
+
/**
|
|
2914
|
+
* Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
|
|
2915
|
+
* exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
|
|
2916
|
+
* suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
|
|
2917
|
+
* stop at a suspension point. Clears the in-memory cancel flag.
|
|
2918
|
+
*/
|
|
2919
|
+
async compensateAndCancel(run, compensations) {
|
|
2920
|
+
this.cancelRequested.delete(run.id);
|
|
2921
|
+
for (let i = compensations.length - 1; i >= 0; i -= 1) {
|
|
2922
|
+
const comp = compensations[i];
|
|
2923
|
+
if (comp) await this.runCompensation(run, comp);
|
|
2924
|
+
}
|
|
2925
|
+
const error = {
|
|
2926
|
+
message: "cancelled"
|
|
2927
|
+
};
|
|
2928
|
+
await this.store.updateRun(run.id, {
|
|
2929
|
+
status: "cancelled",
|
|
2930
|
+
error,
|
|
2931
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2932
|
+
});
|
|
2933
|
+
this.emit({
|
|
2934
|
+
type: "run.failed",
|
|
2935
|
+
runId: run.id,
|
|
2936
|
+
workflow: run.workflow,
|
|
2937
|
+
error
|
|
2938
|
+
});
|
|
2939
|
+
return {
|
|
2940
|
+
runId: run.id,
|
|
2941
|
+
status: "cancelled",
|
|
2942
|
+
error
|
|
2943
|
+
};
|
|
2944
|
+
}
|
|
2851
2945
|
/** The run body, lease held + renewed by {@link execute}. */
|
|
2852
2946
|
async runExecution(run, fn, registered) {
|
|
2947
|
+
if (run.status === "cancelling") this.cancelRequested.add(run.id);
|
|
2853
2948
|
if (run.status === "pending") {
|
|
2854
2949
|
await this.store.updateRun(run.id, {
|
|
2855
2950
|
status: "running",
|
|
@@ -2862,7 +2957,7 @@ var WorkflowEngine = class {
|
|
|
2862
2957
|
workflow: run.workflow
|
|
2863
2958
|
});
|
|
2864
2959
|
}
|
|
2865
|
-
if (registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
2960
|
+
if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
2866
2961
|
const wakeAt = this.singletons.retryWakeAt();
|
|
2867
2962
|
await this.store.updateRun(run.id, {
|
|
2868
2963
|
status: "suspended",
|
|
@@ -2887,12 +2982,14 @@ var WorkflowEngine = class {
|
|
|
2887
2982
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay), run.id, compensations, run.workflow);
|
|
2888
2983
|
try {
|
|
2889
2984
|
const output = await fn(ctx, run.input);
|
|
2985
|
+
if (this.cancelRequested.has(run.id)) return this.compensateAndCancel(run, compensations);
|
|
2890
2986
|
return this.settleRun(run, {
|
|
2891
2987
|
kind: "completed",
|
|
2892
2988
|
output
|
|
2893
2989
|
});
|
|
2894
2990
|
} catch (err) {
|
|
2895
2991
|
if (err instanceof ContinueAsNew) {
|
|
2992
|
+
if (this.cancelRequested.has(run.id)) return this.compensateAndCancel(run, compensations);
|
|
2896
2993
|
await this.store.updateRun(run.id, {
|
|
2897
2994
|
status: "completed",
|
|
2898
2995
|
output: void 0,
|
|
@@ -2916,37 +3013,13 @@ var WorkflowEngine = class {
|
|
|
2916
3013
|
};
|
|
2917
3014
|
}
|
|
2918
3015
|
if (err instanceof WorkflowSuspended) {
|
|
2919
|
-
if (this.cancelRequested.has(run.id))
|
|
2920
|
-
this.cancelRequested.delete(run.id);
|
|
2921
|
-
for (let i = compensations.length - 1; i >= 0; i -= 1) {
|
|
2922
|
-
const comp = compensations[i];
|
|
2923
|
-
if (comp) await this.runCompensation(run, comp);
|
|
2924
|
-
}
|
|
2925
|
-
const error2 = {
|
|
2926
|
-
message: "cancelled"
|
|
2927
|
-
};
|
|
2928
|
-
await this.store.updateRun(run.id, {
|
|
2929
|
-
status: "cancelled",
|
|
2930
|
-
error: error2,
|
|
2931
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2932
|
-
});
|
|
2933
|
-
this.emit({
|
|
2934
|
-
type: "run.failed",
|
|
2935
|
-
runId: run.id,
|
|
2936
|
-
workflow: run.workflow,
|
|
2937
|
-
error: error2
|
|
2938
|
-
});
|
|
2939
|
-
return {
|
|
2940
|
-
runId: run.id,
|
|
2941
|
-
status: "cancelled",
|
|
2942
|
-
error: error2
|
|
2943
|
-
};
|
|
2944
|
-
}
|
|
3016
|
+
if (this.cancelRequested.has(run.id)) return this.compensateAndCancel(run, compensations);
|
|
2945
3017
|
return this.settleRun(run, {
|
|
2946
3018
|
kind: "suspended",
|
|
2947
3019
|
wakeAt: err.wakeAt
|
|
2948
3020
|
});
|
|
2949
3021
|
}
|
|
3022
|
+
if (this.cancelRequested.has(run.id)) return this.compensateAndCancel(run, compensations);
|
|
2950
3023
|
const error = {
|
|
2951
3024
|
message: err instanceof Error ? err.message : String(err),
|
|
2952
3025
|
stack: err instanceof Error ? err.stack : void 0
|
|
@@ -3285,11 +3358,13 @@ var WorkflowEngine = class {
|
|
|
3285
3358
|
}
|
|
3286
3359
|
}
|
|
3287
3360
|
/**
|
|
3288
|
-
* Await
|
|
3289
|
-
* arrives within `timeoutMs`. Each heartbeat (delivered via `transport.onHeartbeat
|
|
3290
|
-
* window, so a worker that keeps beating stays alive past
|
|
3361
|
+
* Await `resultPromise`, but reject with `onTimeout()` if neither it settles nor a heartbeat for `id`
|
|
3362
|
+
* arrives within `timeoutMs`. Each heartbeat (delivered via `transport.onHeartbeat`, looked up in
|
|
3363
|
+
* `heartbeatResets` by `id`) REARMS the window, so a worker that keeps beating stays alive past
|
|
3364
|
+
* `timeoutMs`. Shared by the remote-STEP liveness (keyed by stepId) and the remote-WORKFLOW advance
|
|
3365
|
+
* liveness (keyed by runId); the caller supplies the timeout error and any extra cleanup via `onTimeout`.
|
|
3291
3366
|
*/
|
|
3292
|
-
|
|
3367
|
+
awaitWithLivenessDeadline(id, resultPromise, timeoutMs, onTimeout) {
|
|
3293
3368
|
return new Promise((resolve, reject) => {
|
|
3294
3369
|
let timer;
|
|
3295
3370
|
const cleanup = /* @__PURE__ */ __name(() => {
|
|
@@ -3299,8 +3374,7 @@ var WorkflowEngine = class {
|
|
|
3299
3374
|
const arm = /* @__PURE__ */ __name(() => {
|
|
3300
3375
|
timer = setTimeout(() => {
|
|
3301
3376
|
cleanup();
|
|
3302
|
-
|
|
3303
|
-
reject(new RemoteStepTimeout(id, timeoutMs));
|
|
3377
|
+
reject(onTimeout());
|
|
3304
3378
|
}, timeoutMs);
|
|
3305
3379
|
timer.unref?.();
|
|
3306
3380
|
}, "arm");
|
|
@@ -3318,6 +3392,14 @@ var WorkflowEngine = class {
|
|
|
3318
3392
|
});
|
|
3319
3393
|
});
|
|
3320
3394
|
}
|
|
3395
|
+
/** Remote-STEP liveness: await a worker result, timing out (with the step's pending-map cleanup) to
|
|
3396
|
+
* `RemoteStepTimeout` if it neither lands nor beats within `timeoutMs`. See {@link awaitWithLivenessDeadline}. */
|
|
3397
|
+
awaitWithHeartbeat(id, resultPromise, timeoutMs) {
|
|
3398
|
+
return this.awaitWithLivenessDeadline(id, resultPromise, timeoutMs, () => {
|
|
3399
|
+
this.pending.delete(id);
|
|
3400
|
+
return new RemoteStepTimeout(id, timeoutMs);
|
|
3401
|
+
});
|
|
3402
|
+
}
|
|
3321
3403
|
};
|
|
3322
3404
|
var RemoteStepError = class extends Error {
|
|
3323
3405
|
static {
|
|
@@ -3891,7 +3973,7 @@ var InMemoryStateStore = class {
|
|
|
3891
3973
|
async listIncompleteRuns() {
|
|
3892
3974
|
return [
|
|
3893
3975
|
...this.runs.values()
|
|
3894
|
-
].filter((r) => r.status === "running").map((r) => ({
|
|
3976
|
+
].filter((r) => r.status === "running" || r.status === "cancelling").map((r) => ({
|
|
3895
3977
|
...r
|
|
3896
3978
|
}));
|
|
3897
3979
|
}
|
|
@@ -4083,6 +4165,7 @@ var InMemoryTransport = class {
|
|
|
4083
4165
|
}
|
|
4084
4166
|
handlers = /* @__PURE__ */ new Map();
|
|
4085
4167
|
resultHandler;
|
|
4168
|
+
heartbeatHandler;
|
|
4086
4169
|
stepEventHandler;
|
|
4087
4170
|
controlHandlers = /* @__PURE__ */ new Set();
|
|
4088
4171
|
/** Register a fake worker handler for a step name. */
|
|
@@ -4097,7 +4180,12 @@ var InMemoryTransport = class {
|
|
|
4097
4180
|
onResult(handler) {
|
|
4098
4181
|
this.resultHandler = handler;
|
|
4099
4182
|
}
|
|
4100
|
-
onHeartbeat(
|
|
4183
|
+
onHeartbeat(handler) {
|
|
4184
|
+
this.heartbeatHandler = handler;
|
|
4185
|
+
}
|
|
4186
|
+
/** Test hook: deliver a liveness heartbeat to the engine, as a real broker would on its channel. */
|
|
4187
|
+
async emitHeartbeat(beat) {
|
|
4188
|
+
await this.heartbeatHandler?.(beat);
|
|
4101
4189
|
}
|
|
4102
4190
|
/** Workflow step lifecycle: deliver to the engine's handler (async, mirroring a real broker). */
|
|
4103
4191
|
async dispatchStepEvent(event) {
|