@dudousxd/nestjs-durable-core 0.85.6 → 0.85.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +619 -148
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +125 -19
- package/dist/index.d.ts +125 -19
- package/dist/index.js +619 -148
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -1242,11 +1242,28 @@ interface StateStore {
|
|
|
1242
1242
|
*/
|
|
1243
1243
|
deleteRuns?(runIds: string[]): Promise<void>;
|
|
1244
1244
|
getCheckpoint(runId: string, seq: number): Promise<StepCheckpoint | null>;
|
|
1245
|
+
/**
|
|
1246
|
+
* The run's checkpoints at `seqs`, in `seq` order; a seq with no checkpoint is left out, and an
|
|
1247
|
+
* empty `seqs` reads nothing. The engine reads a batch of one run's step results' checkpoints
|
|
1248
|
+
* through it, one call per batch.
|
|
1249
|
+
*
|
|
1250
|
+
* Optional: a store that omits it gets one {@link getCheckpoint} per seq.
|
|
1251
|
+
*/
|
|
1252
|
+
getCheckpoints?(runId: string, seqs: number[]): Promise<StepCheckpoint[]>;
|
|
1245
1253
|
/**
|
|
1246
1254
|
* Persist a checkpoint and advance the run atomically. Durable semantics depend on this
|
|
1247
1255
|
* being a single transaction; stores without transactions cannot give the strong guarantee.
|
|
1248
1256
|
*/
|
|
1249
1257
|
saveCheckpoint(checkpoint: StepCheckpoint): Promise<void>;
|
|
1258
|
+
/**
|
|
1259
|
+
* {@link saveCheckpoint} for many checkpoints in one call: each is written (inserted, or
|
|
1260
|
+
* overwriting the row at its `(runId, seq)`), in a few statements and one transaction, so a
|
|
1261
|
+
* rejection writes nothing. The entries name distinct rows. The engine writes a step fan's new
|
|
1262
|
+
* `pending` checkpoints through it before dispatching them.
|
|
1263
|
+
*
|
|
1264
|
+
* Optional: a store that omits it gets one {@link saveCheckpoint} per checkpoint.
|
|
1265
|
+
*/
|
|
1266
|
+
saveCheckpoints?(checkpoints: StepCheckpoint[]): Promise<void>;
|
|
1250
1267
|
/**
|
|
1251
1268
|
* Overwrite the checkpoint at `(runId, seq)` with `checkpoint`, but only while the stored row is
|
|
1252
1269
|
* still `pending`; resolves whether it wrote. The engine writes a dispatched step's row from two
|
|
@@ -2328,6 +2345,15 @@ interface RunConsumerOptions extends RunQueueOptions {
|
|
|
2328
2345
|
interface Transport {
|
|
2329
2346
|
/** engine → worker */
|
|
2330
2347
|
dispatch(task: RemoteTask): Promise<void>;
|
|
2348
|
+
/**
|
|
2349
|
+
* {@link dispatch} for many tasks in a few broker calls. Rejects if any task may not have been
|
|
2350
|
+
* dispatched (some may have been: the engine then dispatches the batch again one task at a time,
|
|
2351
|
+
* and a task delivered twice runs twice, as any redelivery does). The engine sends a step fan's
|
|
2352
|
+
* new steps through it.
|
|
2353
|
+
*
|
|
2354
|
+
* Optional: a transport that omits it gets one {@link dispatch} per task.
|
|
2355
|
+
*/
|
|
2356
|
+
dispatchMany?(tasks: RemoteTask[]): Promise<void>;
|
|
2331
2357
|
/** worker → engine: a step finished (ok or error). */
|
|
2332
2358
|
onResult(handler: (result: StepResult) => Promise<void>): void;
|
|
2333
2359
|
/** worker → engine: liveness signal for an in-flight long step. */
|
|
@@ -4679,6 +4705,23 @@ declare class WorkflowEngine {
|
|
|
4679
4705
|
private readonly wakeTurns;
|
|
4680
4706
|
/** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
|
|
4681
4707
|
private readonly leaseRetries;
|
|
4708
|
+
/**
|
|
4709
|
+
* Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
|
|
4710
|
+
* re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
|
|
4711
|
+
* still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
|
|
4712
|
+
*/
|
|
4713
|
+
private readonly unconditionalRechecks;
|
|
4714
|
+
/** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
|
|
4715
|
+
private readonly genericWakeRequested;
|
|
4716
|
+
/** Why each run is being woken on this instance, until the turn that answers it starts. */
|
|
4717
|
+
private readonly wakeReasons;
|
|
4718
|
+
/**
|
|
4719
|
+
* The step fans each executing turn on this instance read with nothing in flight: first seq -> the
|
|
4720
|
+
* highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
|
|
4721
|
+
*/
|
|
4722
|
+
private readonly turnSettledFans;
|
|
4723
|
+
/** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
|
|
4724
|
+
private readonly turnServedWakes;
|
|
4682
4725
|
/**
|
|
4683
4726
|
* Every run turn and child start this instance takes on itself, under one budget of
|
|
4684
4727
|
* {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
|
|
@@ -4946,13 +4989,17 @@ declare class WorkflowEngine {
|
|
|
4946
4989
|
*
|
|
4947
4990
|
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
4948
4991
|
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
4949
|
-
* few timer ticks without a long query or a big delete.
|
|
4950
|
-
*
|
|
4951
|
-
*
|
|
4952
|
-
*
|
|
4953
|
-
*
|
|
4954
|
-
|
|
4955
|
-
|
|
4992
|
+
* few timer ticks without a long query or a big delete. Each page is decided set-based (see
|
|
4993
|
+
* `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
|
|
4994
|
+
* read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
|
|
4995
|
+
* {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
|
|
4996
|
+
* call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
|
|
4997
|
+
* before the next one. Call it every tick (the NestJS timer poller does); a call made while
|
|
4998
|
+
* another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
|
|
4999
|
+
* carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
|
|
5000
|
+
* store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
5001
|
+
*/
|
|
5002
|
+
sweepBufferedSignals(now?: number, budgetMs?: number): Promise<number>;
|
|
4956
5003
|
/**
|
|
4957
5004
|
* Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
|
|
4958
5005
|
* either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
|
|
@@ -5003,6 +5050,8 @@ declare class WorkflowEngine {
|
|
|
5003
5050
|
* terminal too.
|
|
5004
5051
|
*/
|
|
5005
5052
|
resume(runId: string): Promise<RunResult>;
|
|
5053
|
+
/** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
|
|
5054
|
+
private noteWakeReason;
|
|
5006
5055
|
/**
|
|
5007
5056
|
* {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
|
|
5008
5057
|
* result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
|
|
@@ -5742,19 +5791,48 @@ declare class WorkflowEngine {
|
|
|
5742
5791
|
/**
|
|
5743
5792
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
5744
5793
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
|
5745
|
-
* (a signal, a child's completion). The request is a buffered
|
|
5746
|
-
* on
|
|
5794
|
+
* (a signal, a child's completion). The request is a buffered event under a name no workflow waits
|
|
5795
|
+
* on. It stays until a turn that started after it has settled: the holder looks at it after it
|
|
5796
|
+
* releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
|
|
5747
5797
|
*
|
|
5748
5798
|
* Written first and looked at second, against a holder that releases first and looks second:
|
|
5749
5799
|
* whichever look runs last sees the other side's write. So when the run is no longer executing by
|
|
5750
|
-
* the time this re-reads it, the holder may have looked already, and this side
|
|
5751
|
-
*
|
|
5800
|
+
* the time this re-reads it, the holder may have looked already, and this side wakes the run
|
|
5801
|
+
* itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
|
|
5802
|
+
* made due when the holder's lease runs out, so a holder that goes down before it settles leaves
|
|
5803
|
+
* the run to any instance's timer poll; a holder that settles sets the run's timer itself.
|
|
5752
5804
|
*/
|
|
5753
5805
|
private requestWakeAfterTurn;
|
|
5754
|
-
/**
|
|
5806
|
+
/**
|
|
5807
|
+
* After a turn that suspended released its lease: drop the wakes it served, and run again if a
|
|
5808
|
+
* wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
|
|
5809
|
+
* read that fan with nothing in flight, at least as far as the batch that opened its barrier.
|
|
5810
|
+
* The run is made due before it is woken, so if this instance goes down before that turn runs,
|
|
5811
|
+
* the timer poll of another instance runs it once this instance's hold on it would have lapsed.
|
|
5812
|
+
*/
|
|
5755
5813
|
private honorWakeRequests;
|
|
5756
|
-
/**
|
|
5757
|
-
|
|
5814
|
+
/**
|
|
5815
|
+
* The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
|
|
5816
|
+
* signals, which are taken as they are read.
|
|
5817
|
+
*/
|
|
5818
|
+
private listWakeRequests;
|
|
5819
|
+
/** Delete `requests` (the ones kept as buffered events). */
|
|
5820
|
+
private dropWakeRequests;
|
|
5821
|
+
/**
|
|
5822
|
+
* Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
|
|
5823
|
+
* turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
|
|
5824
|
+
* until then stay for whoever runs the run next.
|
|
5825
|
+
*/
|
|
5826
|
+
private noteServedWakes;
|
|
5827
|
+
/**
|
|
5828
|
+
* Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
|
|
5829
|
+
* was kept for the run during the turn that the turn did not see, keep the run due when the lease
|
|
5830
|
+
* runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
|
|
5831
|
+
* may go down before it looks at the wake once it has released the lease.
|
|
5832
|
+
*/
|
|
5833
|
+
private keepDueForKeptWakes;
|
|
5834
|
+
/** Bring a suspended run's timer forward to `at` (never later than the one it has). */
|
|
5835
|
+
private makeDueBy;
|
|
5758
5836
|
/**
|
|
5759
5837
|
* A wake that found the run leased by another instance is dropped, and normally that is fine: the
|
|
5760
5838
|
* holder's turn either sees what the wake wrote, or re-checks once it releases the lease
|
|
@@ -6101,6 +6179,11 @@ declare class WorkflowEngine {
|
|
|
6101
6179
|
* capable+compatible worker has appeared, else re-parks with a fresh `wakeAt` (design §7.5). */
|
|
6102
6180
|
private dueBlockedRuns;
|
|
6103
6181
|
private callRemote;
|
|
6182
|
+
/**
|
|
6183
|
+
* Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
|
|
6184
|
+
* suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
|
|
6185
|
+
*/
|
|
6186
|
+
private stepIsWaiting;
|
|
6104
6187
|
/**
|
|
6105
6188
|
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
6106
6189
|
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
@@ -6113,7 +6196,11 @@ declare class WorkflowEngine {
|
|
|
6113
6196
|
* barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
|
|
6114
6197
|
*/
|
|
6115
6198
|
private callRemoteAll;
|
|
6116
|
-
/**
|
|
6199
|
+
/**
|
|
6200
|
+
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
6201
|
+
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
6202
|
+
* batch, where the store and transport have them.
|
|
6203
|
+
*/
|
|
6117
6204
|
private sendStepFan;
|
|
6118
6205
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
6119
6206
|
private noteSettledHere;
|
|
@@ -6128,6 +6215,12 @@ declare class WorkflowEngine {
|
|
|
6128
6215
|
* call rejecting means it wrote nothing, so the entries are then written one by one.
|
|
6129
6216
|
*/
|
|
6130
6217
|
private savePendingCheckpoints;
|
|
6218
|
+
/**
|
|
6219
|
+
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
6220
|
+
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
6221
|
+
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
6222
|
+
*/
|
|
6223
|
+
private readCheckpoints;
|
|
6131
6224
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
6132
6225
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
6133
6226
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -6147,9 +6240,10 @@ declare class WorkflowEngine {
|
|
|
6147
6240
|
*
|
|
6148
6241
|
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
6149
6242
|
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
6150
|
-
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked
|
|
6151
|
-
*
|
|
6152
|
-
* transport acknowledges nothing that is not durable yet.
|
|
6243
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
|
|
6244
|
+
* by a batch with no more of the fan's results queued behind it. A result resolves only once its
|
|
6245
|
+
* own write has committed, so the transport acknowledges nothing that is not durable yet. How
|
|
6246
|
+
* large a batch gets is bounded by how many results the transport hands over at once.
|
|
6153
6247
|
*/
|
|
6154
6248
|
private completeRemoteResult;
|
|
6155
6249
|
/** Settle `queue` (a run's queued results) batch by batch until it is empty, then retire it. */
|
|
@@ -6158,6 +6252,11 @@ declare class WorkflowEngine {
|
|
|
6158
6252
|
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
6159
6253
|
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
6160
6254
|
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
6255
|
+
*
|
|
6256
|
+
* `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
|
|
6257
|
+
* run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
|
|
6258
|
+
* asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
|
|
6259
|
+
* under a backlog the barrier is read once per drained queue rather than once per batch.
|
|
6161
6260
|
*/
|
|
6162
6261
|
private completeRemoteResults;
|
|
6163
6262
|
/**
|
|
@@ -6172,7 +6271,8 @@ declare class WorkflowEngine {
|
|
|
6172
6271
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
6173
6272
|
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
6174
6273
|
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
6175
|
-
* re-suspends, a lost one strands the run.
|
|
6274
|
+
* re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
|
|
6275
|
+
* that result (see {@link completeRemoteResults}).
|
|
6176
6276
|
*/
|
|
6177
6277
|
private stepFanStillRunning;
|
|
6178
6278
|
/** How many steps of a step fan's seq range are still `pending`. */
|
|
@@ -6893,6 +6993,10 @@ declare class CodecStateStore implements StateStore {
|
|
|
6893
6993
|
readonly saveCheckpointIfPending?: (checkpoint: StepCheckpoint) => Promise<boolean>;
|
|
6894
6994
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
6895
6995
|
readonly saveCheckpointsIfPending?: (checkpoints: StepCheckpoint[]) => Promise<boolean[]>;
|
|
6996
|
+
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
6997
|
+
readonly saveCheckpoints?: (checkpoints: StepCheckpoint[]) => Promise<void>;
|
|
6998
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
6999
|
+
readonly getCheckpoints?: (runId: string, seqs: number[]) => Promise<StepCheckpoint[]>;
|
|
6896
7000
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
6897
7001
|
readonly countCheckpoints?: (runId: string, fromSeq: number, toSeq: number, statuses: StepCheckpoint['status'][]) => Promise<number>;
|
|
6898
7002
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -7438,7 +7542,9 @@ declare class InMemoryStateStore implements StateStore {
|
|
|
7438
7542
|
deleteRuns(runIds: string[]): Promise<void>;
|
|
7439
7543
|
pruneTerminalRuns(policy: RetentionPolicy, nowMs: number, limit: number): Promise<number>;
|
|
7440
7544
|
getCheckpoint(runId: string, seq: number): Promise<StepCheckpoint | null>;
|
|
7545
|
+
getCheckpoints(runId: string, seqs: number[]): Promise<StepCheckpoint[]>;
|
|
7441
7546
|
saveCheckpoint(checkpoint: StepCheckpoint): Promise<void>;
|
|
7547
|
+
saveCheckpoints(checkpoints: StepCheckpoint[]): Promise<void>;
|
|
7442
7548
|
saveCheckpointIfPending(checkpoint: StepCheckpoint): Promise<boolean>;
|
|
7443
7549
|
saveCheckpointsIfPending(checkpoints: StepCheckpoint[]): Promise<boolean[]>;
|
|
7444
7550
|
listOrphanedRuns(nowMs: number, limit: number, namespace?: string, ownedBy?: OwnedLeaseReclaim): Promise<WorkflowRun[]>;
|
package/dist/index.d.ts
CHANGED
|
@@ -1242,11 +1242,28 @@ interface StateStore {
|
|
|
1242
1242
|
*/
|
|
1243
1243
|
deleteRuns?(runIds: string[]): Promise<void>;
|
|
1244
1244
|
getCheckpoint(runId: string, seq: number): Promise<StepCheckpoint | null>;
|
|
1245
|
+
/**
|
|
1246
|
+
* The run's checkpoints at `seqs`, in `seq` order; a seq with no checkpoint is left out, and an
|
|
1247
|
+
* empty `seqs` reads nothing. The engine reads a batch of one run's step results' checkpoints
|
|
1248
|
+
* through it, one call per batch.
|
|
1249
|
+
*
|
|
1250
|
+
* Optional: a store that omits it gets one {@link getCheckpoint} per seq.
|
|
1251
|
+
*/
|
|
1252
|
+
getCheckpoints?(runId: string, seqs: number[]): Promise<StepCheckpoint[]>;
|
|
1245
1253
|
/**
|
|
1246
1254
|
* Persist a checkpoint and advance the run atomically. Durable semantics depend on this
|
|
1247
1255
|
* being a single transaction; stores without transactions cannot give the strong guarantee.
|
|
1248
1256
|
*/
|
|
1249
1257
|
saveCheckpoint(checkpoint: StepCheckpoint): Promise<void>;
|
|
1258
|
+
/**
|
|
1259
|
+
* {@link saveCheckpoint} for many checkpoints in one call: each is written (inserted, or
|
|
1260
|
+
* overwriting the row at its `(runId, seq)`), in a few statements and one transaction, so a
|
|
1261
|
+
* rejection writes nothing. The entries name distinct rows. The engine writes a step fan's new
|
|
1262
|
+
* `pending` checkpoints through it before dispatching them.
|
|
1263
|
+
*
|
|
1264
|
+
* Optional: a store that omits it gets one {@link saveCheckpoint} per checkpoint.
|
|
1265
|
+
*/
|
|
1266
|
+
saveCheckpoints?(checkpoints: StepCheckpoint[]): Promise<void>;
|
|
1250
1267
|
/**
|
|
1251
1268
|
* Overwrite the checkpoint at `(runId, seq)` with `checkpoint`, but only while the stored row is
|
|
1252
1269
|
* still `pending`; resolves whether it wrote. The engine writes a dispatched step's row from two
|
|
@@ -2328,6 +2345,15 @@ interface RunConsumerOptions extends RunQueueOptions {
|
|
|
2328
2345
|
interface Transport {
|
|
2329
2346
|
/** engine → worker */
|
|
2330
2347
|
dispatch(task: RemoteTask): Promise<void>;
|
|
2348
|
+
/**
|
|
2349
|
+
* {@link dispatch} for many tasks in a few broker calls. Rejects if any task may not have been
|
|
2350
|
+
* dispatched (some may have been: the engine then dispatches the batch again one task at a time,
|
|
2351
|
+
* and a task delivered twice runs twice, as any redelivery does). The engine sends a step fan's
|
|
2352
|
+
* new steps through it.
|
|
2353
|
+
*
|
|
2354
|
+
* Optional: a transport that omits it gets one {@link dispatch} per task.
|
|
2355
|
+
*/
|
|
2356
|
+
dispatchMany?(tasks: RemoteTask[]): Promise<void>;
|
|
2331
2357
|
/** worker → engine: a step finished (ok or error). */
|
|
2332
2358
|
onResult(handler: (result: StepResult) => Promise<void>): void;
|
|
2333
2359
|
/** worker → engine: liveness signal for an in-flight long step. */
|
|
@@ -4679,6 +4705,23 @@ declare class WorkflowEngine {
|
|
|
4679
4705
|
private readonly wakeTurns;
|
|
4680
4706
|
/** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
|
|
4681
4707
|
private readonly leaseRetries;
|
|
4708
|
+
/**
|
|
4709
|
+
* Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
|
|
4710
|
+
* re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
|
|
4711
|
+
* still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
|
|
4712
|
+
*/
|
|
4713
|
+
private readonly unconditionalRechecks;
|
|
4714
|
+
/** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
|
|
4715
|
+
private readonly genericWakeRequested;
|
|
4716
|
+
/** Why each run is being woken on this instance, until the turn that answers it starts. */
|
|
4717
|
+
private readonly wakeReasons;
|
|
4718
|
+
/**
|
|
4719
|
+
* The step fans each executing turn on this instance read with nothing in flight: first seq -> the
|
|
4720
|
+
* highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
|
|
4721
|
+
*/
|
|
4722
|
+
private readonly turnSettledFans;
|
|
4723
|
+
/** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
|
|
4724
|
+
private readonly turnServedWakes;
|
|
4682
4725
|
/**
|
|
4683
4726
|
* Every run turn and child start this instance takes on itself, under one budget of
|
|
4684
4727
|
* {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
|
|
@@ -4946,13 +4989,17 @@ declare class WorkflowEngine {
|
|
|
4946
4989
|
*
|
|
4947
4990
|
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
4948
4991
|
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
4949
|
-
* few timer ticks without a long query or a big delete.
|
|
4950
|
-
*
|
|
4951
|
-
*
|
|
4952
|
-
*
|
|
4953
|
-
*
|
|
4954
|
-
|
|
4955
|
-
|
|
4992
|
+
* few timer ticks without a long query or a big delete. Each page is decided set-based (see
|
|
4993
|
+
* `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
|
|
4994
|
+
* read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
|
|
4995
|
+
* {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
|
|
4996
|
+
* call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
|
|
4997
|
+
* before the next one. Call it every tick (the NestJS timer poller does); a call made while
|
|
4998
|
+
* another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
|
|
4999
|
+
* carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
|
|
5000
|
+
* store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
5001
|
+
*/
|
|
5002
|
+
sweepBufferedSignals(now?: number, budgetMs?: number): Promise<number>;
|
|
4956
5003
|
/**
|
|
4957
5004
|
* Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
|
|
4958
5005
|
* either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
|
|
@@ -5003,6 +5050,8 @@ declare class WorkflowEngine {
|
|
|
5003
5050
|
* terminal too.
|
|
5004
5051
|
*/
|
|
5005
5052
|
resume(runId: string): Promise<RunResult>;
|
|
5053
|
+
/** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
|
|
5054
|
+
private noteWakeReason;
|
|
5006
5055
|
/**
|
|
5007
5056
|
* {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
|
|
5008
5057
|
* result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
|
|
@@ -5742,19 +5791,48 @@ declare class WorkflowEngine {
|
|
|
5742
5791
|
/**
|
|
5743
5792
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
5744
5793
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
|
5745
|
-
* (a signal, a child's completion). The request is a buffered
|
|
5746
|
-
* on
|
|
5794
|
+
* (a signal, a child's completion). The request is a buffered event under a name no workflow waits
|
|
5795
|
+
* on. It stays until a turn that started after it has settled: the holder looks at it after it
|
|
5796
|
+
* releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
|
|
5747
5797
|
*
|
|
5748
5798
|
* Written first and looked at second, against a holder that releases first and looks second:
|
|
5749
5799
|
* whichever look runs last sees the other side's write. So when the run is no longer executing by
|
|
5750
|
-
* the time this re-reads it, the holder may have looked already, and this side
|
|
5751
|
-
*
|
|
5800
|
+
* the time this re-reads it, the holder may have looked already, and this side wakes the run
|
|
5801
|
+
* itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
|
|
5802
|
+
* made due when the holder's lease runs out, so a holder that goes down before it settles leaves
|
|
5803
|
+
* the run to any instance's timer poll; a holder that settles sets the run's timer itself.
|
|
5752
5804
|
*/
|
|
5753
5805
|
private requestWakeAfterTurn;
|
|
5754
|
-
/**
|
|
5806
|
+
/**
|
|
5807
|
+
* After a turn that suspended released its lease: drop the wakes it served, and run again if a
|
|
5808
|
+
* wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
|
|
5809
|
+
* read that fan with nothing in flight, at least as far as the batch that opened its barrier.
|
|
5810
|
+
* The run is made due before it is woken, so if this instance goes down before that turn runs,
|
|
5811
|
+
* the timer poll of another instance runs it once this instance's hold on it would have lapsed.
|
|
5812
|
+
*/
|
|
5755
5813
|
private honorWakeRequests;
|
|
5756
|
-
/**
|
|
5757
|
-
|
|
5814
|
+
/**
|
|
5815
|
+
* The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
|
|
5816
|
+
* signals, which are taken as they are read.
|
|
5817
|
+
*/
|
|
5818
|
+
private listWakeRequests;
|
|
5819
|
+
/** Delete `requests` (the ones kept as buffered events). */
|
|
5820
|
+
private dropWakeRequests;
|
|
5821
|
+
/**
|
|
5822
|
+
* Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
|
|
5823
|
+
* turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
|
|
5824
|
+
* until then stay for whoever runs the run next.
|
|
5825
|
+
*/
|
|
5826
|
+
private noteServedWakes;
|
|
5827
|
+
/**
|
|
5828
|
+
* Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
|
|
5829
|
+
* was kept for the run during the turn that the turn did not see, keep the run due when the lease
|
|
5830
|
+
* runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
|
|
5831
|
+
* may go down before it looks at the wake once it has released the lease.
|
|
5832
|
+
*/
|
|
5833
|
+
private keepDueForKeptWakes;
|
|
5834
|
+
/** Bring a suspended run's timer forward to `at` (never later than the one it has). */
|
|
5835
|
+
private makeDueBy;
|
|
5758
5836
|
/**
|
|
5759
5837
|
* A wake that found the run leased by another instance is dropped, and normally that is fine: the
|
|
5760
5838
|
* holder's turn either sees what the wake wrote, or re-checks once it releases the lease
|
|
@@ -6101,6 +6179,11 @@ declare class WorkflowEngine {
|
|
|
6101
6179
|
* capable+compatible worker has appeared, else re-parks with a fresh `wakeAt` (design §7.5). */
|
|
6102
6180
|
private dueBlockedRuns;
|
|
6103
6181
|
private callRemote;
|
|
6182
|
+
/**
|
|
6183
|
+
* Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
|
|
6184
|
+
* suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
|
|
6185
|
+
*/
|
|
6186
|
+
private stepIsWaiting;
|
|
6104
6187
|
/**
|
|
6105
6188
|
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
6106
6189
|
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
@@ -6113,7 +6196,11 @@ declare class WorkflowEngine {
|
|
|
6113
6196
|
* barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
|
|
6114
6197
|
*/
|
|
6115
6198
|
private callRemoteAll;
|
|
6116
|
-
/**
|
|
6199
|
+
/**
|
|
6200
|
+
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
6201
|
+
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
6202
|
+
* batch, where the store and transport have them.
|
|
6203
|
+
*/
|
|
6117
6204
|
private sendStepFan;
|
|
6118
6205
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
6119
6206
|
private noteSettledHere;
|
|
@@ -6128,6 +6215,12 @@ declare class WorkflowEngine {
|
|
|
6128
6215
|
* call rejecting means it wrote nothing, so the entries are then written one by one.
|
|
6129
6216
|
*/
|
|
6130
6217
|
private savePendingCheckpoints;
|
|
6218
|
+
/**
|
|
6219
|
+
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
6220
|
+
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
6221
|
+
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
6222
|
+
*/
|
|
6223
|
+
private readCheckpoints;
|
|
6131
6224
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
6132
6225
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
6133
6226
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -6147,9 +6240,10 @@ declare class WorkflowEngine {
|
|
|
6147
6240
|
*
|
|
6148
6241
|
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
6149
6242
|
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
6150
|
-
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked
|
|
6151
|
-
*
|
|
6152
|
-
* transport acknowledges nothing that is not durable yet.
|
|
6243
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
|
|
6244
|
+
* by a batch with no more of the fan's results queued behind it. A result resolves only once its
|
|
6245
|
+
* own write has committed, so the transport acknowledges nothing that is not durable yet. How
|
|
6246
|
+
* large a batch gets is bounded by how many results the transport hands over at once.
|
|
6153
6247
|
*/
|
|
6154
6248
|
private completeRemoteResult;
|
|
6155
6249
|
/** Settle `queue` (a run's queued results) batch by batch until it is empty, then retire it. */
|
|
@@ -6158,6 +6252,11 @@ declare class WorkflowEngine {
|
|
|
6158
6252
|
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
6159
6253
|
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
6160
6254
|
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
6255
|
+
*
|
|
6256
|
+
* `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
|
|
6257
|
+
* run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
|
|
6258
|
+
* asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
|
|
6259
|
+
* under a backlog the barrier is read once per drained queue rather than once per batch.
|
|
6161
6260
|
*/
|
|
6162
6261
|
private completeRemoteResults;
|
|
6163
6262
|
/**
|
|
@@ -6172,7 +6271,8 @@ declare class WorkflowEngine {
|
|
|
6172
6271
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
6173
6272
|
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
6174
6273
|
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
6175
|
-
* re-suspends, a lost one strands the run.
|
|
6274
|
+
* re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
|
|
6275
|
+
* that result (see {@link completeRemoteResults}).
|
|
6176
6276
|
*/
|
|
6177
6277
|
private stepFanStillRunning;
|
|
6178
6278
|
/** How many steps of a step fan's seq range are still `pending`. */
|
|
@@ -6893,6 +6993,10 @@ declare class CodecStateStore implements StateStore {
|
|
|
6893
6993
|
readonly saveCheckpointIfPending?: (checkpoint: StepCheckpoint) => Promise<boolean>;
|
|
6894
6994
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
6895
6995
|
readonly saveCheckpointsIfPending?: (checkpoints: StepCheckpoint[]) => Promise<boolean[]>;
|
|
6996
|
+
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
6997
|
+
readonly saveCheckpoints?: (checkpoints: StepCheckpoint[]) => Promise<void>;
|
|
6998
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
6999
|
+
readonly getCheckpoints?: (runId: string, seqs: number[]) => Promise<StepCheckpoint[]>;
|
|
6896
7000
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
6897
7001
|
readonly countCheckpoints?: (runId: string, fromSeq: number, toSeq: number, statuses: StepCheckpoint['status'][]) => Promise<number>;
|
|
6898
7002
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -7438,7 +7542,9 @@ declare class InMemoryStateStore implements StateStore {
|
|
|
7438
7542
|
deleteRuns(runIds: string[]): Promise<void>;
|
|
7439
7543
|
pruneTerminalRuns(policy: RetentionPolicy, nowMs: number, limit: number): Promise<number>;
|
|
7440
7544
|
getCheckpoint(runId: string, seq: number): Promise<StepCheckpoint | null>;
|
|
7545
|
+
getCheckpoints(runId: string, seqs: number[]): Promise<StepCheckpoint[]>;
|
|
7441
7546
|
saveCheckpoint(checkpoint: StepCheckpoint): Promise<void>;
|
|
7547
|
+
saveCheckpoints(checkpoints: StepCheckpoint[]): Promise<void>;
|
|
7442
7548
|
saveCheckpointIfPending(checkpoint: StepCheckpoint): Promise<boolean>;
|
|
7443
7549
|
saveCheckpointsIfPending(checkpoints: StepCheckpoint[]): Promise<boolean[]>;
|
|
7444
7550
|
listOrphanedRuns(nowMs: number, limit: number, namespace?: string, ownedBy?: OwnedLeaseReclaim): Promise<WorkflowRun[]>;
|