@dudousxd/nestjs-durable-core 0.105.0 → 0.106.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -3496,6 +3496,11 @@ var RemoteWorkflowExecutor = class {
3496
3496
  };
3497
3497
  await this.transport.dispatchWorkflowTask(task);
3498
3498
  }
3499
+ /** Where the broker has the task dispatched under `taskId`, when the transport can tell. */
3500
+ async taskState(_run, taskId) {
3501
+ if (!this.transport.workflowTaskState) return "unknown";
3502
+ return this.transport.workflowTaskState(tenantGroup(sanitizeQueueToken(this.name), this.partition), taskId);
3503
+ }
3499
3504
  };
3500
3505
 
3501
3506
  // src/run-dispatch-batcher.ts
@@ -7305,6 +7310,10 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
7305
7310
  __name(createWorkflowCtx, "createWorkflowCtx");
7306
7311
 
7307
7312
  // src/engine.ts
7313
+ function isRequeuedStepFailure(cp) {
7314
+ return cp.status === "failed" && cp.attempts === 0 && cp.seq >= 0 && (cp.kind === "local" || cp.kind === "remote");
7315
+ }
7316
+ __name(isRequeuedStepFailure, "isRequeuedStepFailure");
7308
7317
  function mergeTags(staticTags, runTags) {
7309
7318
  if (!staticTags?.length && !runTags?.length) return void 0;
7310
7319
  return [
@@ -7316,6 +7325,20 @@ function mergeTags(staticTags, runTags) {
7316
7325
  }
7317
7326
  __name(mergeTags, "mergeTags");
7318
7327
  var BREAKPOINT = "breakpoint";
7328
+ var LOST_TURN_SUFFIX = ":lost";
7329
+ var GRACED_TURN_SUFFIX = ":grace";
7330
+ function awaitsDecision(run, decision) {
7331
+ const marker = run.awaitingDecisionTaskId;
7332
+ return marker !== void 0 && awaitedTaskId(marker) === decision.taskId;
7333
+ }
7334
+ __name(awaitsDecision, "awaitsDecision");
7335
+ function awaitedTaskId(marker) {
7336
+ let id = marker;
7337
+ if (id.endsWith(LOST_TURN_SUFFIX)) id = id.slice(0, -LOST_TURN_SUFFIX.length);
7338
+ if (id.endsWith(GRACED_TURN_SUFFIX)) id = id.slice(0, -GRACED_TURN_SUFFIX.length);
7339
+ return id;
7340
+ }
7341
+ __name(awaitedTaskId, "awaitedTaskId");
7319
7342
  var PENDING_POLL_BATCH = 100;
7320
7343
  var DEFAULT_STICKY_RUNS = 1e3;
7321
7344
  var DEFAULT_STICKY_IDLE_MS = 3e5;
@@ -7585,6 +7608,9 @@ var WorkflowEngine = class _WorkflowEngine {
7585
7608
  * `clock() + leaseMs`, so at or after it). See {@link orphanedRuns}.
7586
7609
  */
7587
7610
  ownLeaseCutoffMs;
7611
+ /** When this engine was built (clock ms): {@link noteLapsedTurn} counts no lost turn within one
7612
+ * turn window of it, while live workers' beats catch up with deadlines that went stale. */
7613
+ bootedAtMs;
7588
7614
  maxRecoveryAttempts;
7589
7615
  unregisteredWorkflowGraceMs;
7590
7616
  remoteAdvanceSilenceMs;
@@ -7814,6 +7840,7 @@ var WorkflowEngine = class _WorkflowEngine {
7814
7840
  this.namespace = poolNamespace(deps.namespace);
7815
7841
  this.leaseMs = deps.leaseMs ?? 3e4;
7816
7842
  this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
7843
+ this.bootedAtMs = this.clock();
7817
7844
  this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
7818
7845
  const grace = deps.unregisteredWorkflowGraceMs;
7819
7846
  this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
@@ -8191,6 +8218,9 @@ var WorkflowEngine = class _WorkflowEngine {
8191
8218
  return;
8192
8219
  }
8193
8220
  if (dispatch.reason !== "wake" || TERMINAL_RUN_STATUSES.includes(run.status)) return;
8221
+ if (dispatch.awaiting !== void 0 && (run.awaitingDecisionTaskId === void 0 || awaitedTaskId(run.awaitingDecisionTaskId) !== dispatch.awaiting)) {
8222
+ return;
8223
+ }
8194
8224
  const nowMs = this.clock();
8195
8225
  if (run.wakeAt !== void 0 && run.wakeAt <= nowMs) {
8196
8226
  const waiters = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -8242,6 +8272,7 @@ var WorkflowEngine = class _WorkflowEngine {
8242
8272
  */
8243
8273
  async handOffDue(runs, nowMs, settled) {
8244
8274
  const claimed = [];
8275
+ const awaiting = /* @__PURE__ */ new Map();
8245
8276
  for (const run of runs) {
8246
8277
  if (this.draining) break;
8247
8278
  this.markClaim(run.id);
@@ -8256,16 +8287,26 @@ var WorkflowEngine = class _WorkflowEngine {
8256
8287
  continue;
8257
8288
  }
8258
8289
  try {
8290
+ const seen = {};
8291
+ const handled = await this.noteLapsedTurn(run.id, nowMs, seen);
8292
+ if (handled) {
8293
+ if (handled.status === "dead") settled?.set(run.id, handled);
8294
+ continue;
8295
+ }
8259
8296
  const patch = {
8260
8297
  wakeAt: nowMs + this.leaseMs
8261
8298
  };
8299
+ let won = true;
8262
8300
  if (this.store.updateRunIf) {
8263
- if (await this.store.updateRunIf(run.id, [
8301
+ won = await this.store.updateRunIf(run.id, [
8264
8302
  run.status
8265
- ], patch)) claimed.push(run);
8303
+ ], patch);
8266
8304
  } else {
8267
8305
  await this.store.updateRun(run.id, patch);
8306
+ }
8307
+ if (won) {
8268
8308
  claimed.push(run);
8309
+ if (seen.marker !== void 0) awaiting.set(run.id, awaitedTaskId(seen.marker));
8269
8310
  }
8270
8311
  } finally {
8271
8312
  await this.store.releaseRunLock(run.id, this.instanceId);
@@ -8276,13 +8317,19 @@ var WorkflowEngine = class _WorkflowEngine {
8276
8317
  }
8277
8318
  try {
8278
8319
  for (let at = 0; at < claimed.length; at += RUN_DISPATCH_BATCH) {
8279
- await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => ({
8280
- runId: run.id,
8281
- reason: "wake",
8282
- ...run.priority !== void 0 ? {
8283
- priority: run.priority
8284
- } : {}
8285
- })), {
8320
+ await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => {
8321
+ const turn = awaiting.get(run.id);
8322
+ return {
8323
+ runId: run.id,
8324
+ reason: "wake",
8325
+ ...run.priority !== void 0 ? {
8326
+ priority: run.priority
8327
+ } : {},
8328
+ ...turn !== void 0 ? {
8329
+ awaiting: turn
8330
+ } : {}
8331
+ };
8332
+ }), {
8286
8333
  namespace: this.namespace
8287
8334
  });
8288
8335
  }
@@ -11356,28 +11403,39 @@ var WorkflowEngine = class _WorkflowEngine {
11356
11403
  * `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
11357
11404
  * result to skip the resume, or `undefined` to proceed.
11358
11405
  */
11359
- async countRecovery(run) {
11406
+ async countRecovery(run, extra = {}) {
11360
11407
  const attempts = (run.recoveryAttempts ?? 0) + 1;
11408
+ const updateIf = /* @__PURE__ */ __name((patch) => (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11409
+ run.status
11410
+ ], patch), "updateIf");
11411
+ const moved = /* @__PURE__ */ __name(async () => {
11412
+ await this.store.releaseRunLock(run.id);
11413
+ return this.echoCurrentStatus(run.id);
11414
+ }, "moved");
11361
11415
  if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
11362
11416
  const error2 = {
11363
11417
  message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
11364
11418
  code: "max_recovery_attempts"
11365
11419
  };
11366
- await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
11420
+ const written = await updateIf(await this.withLifecycleMarker(run, {
11367
11421
  status: "dead",
11368
11422
  error: error2,
11369
11423
  updatedAt: /* @__PURE__ */ new Date()
11370
11424
  }));
11425
+ if (!written) return moved();
11371
11426
  await this.store.releaseRunLock(run.id);
11372
11427
  return this.announceDead({
11373
11428
  ...run,
11374
11429
  recoveryAttempts: attempts
11375
11430
  }, error2);
11376
11431
  }
11377
- await this.store.updateRun(run.id, {
11432
+ if (!await updateIf({
11433
+ ...extra,
11378
11434
  recoveryAttempts: attempts,
11379
11435
  updatedAt: /* @__PURE__ */ new Date()
11380
- });
11436
+ })) {
11437
+ return moved();
11438
+ }
11381
11439
  return void 0;
11382
11440
  }
11383
11441
  /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
@@ -11524,7 +11582,87 @@ var WorkflowEngine = class _WorkflowEngine {
11524
11582
  return this.resumeLeased([
11525
11583
  ...due,
11526
11584
  ...blocked
11527
- ], nowMs, (run) => this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11585
+ ], nowMs, async (run) => await this.noteLapsedTurn(run.id, nowMs) ?? this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11586
+ }
11587
+ /**
11588
+ * Called under the run's lease by the due-timer pass, for a run that THIS instance would re-drive.
11589
+ * A run suspended awaiting a DISPATCHED workflow turn's decision (`awaitingDecisionTaskId`) whose
11590
+ * re-drive deadline (`wakeAt`) has passed lost that turn: no decision came back and the worker
11591
+ * stopped beating (every beat pushes `wakeAt` forward, see {@link rearmDecisionDeadline}), so the
11592
+ * worker running it died, or the task was lost with it. That is the dispatched form of a crashed
11593
+ * inline turn, so it spends the same {@link maxRecoveryAttempts} budget through
11594
+ * {@link countRecovery}: a turn that keeps killing its worker ends `dead` (with the same
11595
+ * `run.failed` announcement) instead of being re-dispatched, and redelivered, for ever.
11596
+ *
11597
+ * Counted at most once per awaited turn: the count rewrites the marker to its `:lost` form, which
11598
+ * no later pass counts again however long the re-drive waits on the run queue, and the re-drive's
11599
+ * dispatch replaces it with the next turn's id. Not counted, and the re-drive left for later:
11600
+ * - while the broker still holds the task unstarted (a backed-up queue, not a dead worker), or
11601
+ * - within one turn window of this engine's boot, when beats from live workers have not yet had
11602
+ * the chance to push a deadline that went stale while the engine was down. Once per turn (the
11603
+ * `:grace` marker): a lapse after that takes the normal path, whichever instance judges it.
11604
+ * A late decision of a counted turn is still applied while the re-drive has not dispatched yet
11605
+ * (see `awaitsDecision`), and gives the attempt back.
11606
+ * Never counted by an instance that cannot resolve the workflow (it re-drives nothing), nor for a
11607
+ * decision that arrives, a worker that keeps beating, or any write a healthy turn makes on its way.
11608
+ *
11609
+ * Re-reads under the lease: the listing may predate a decision applied since. Returns a result when
11610
+ * the run was dead-lettered or left waiting (the lease released either way), else `undefined` for
11611
+ * the caller to re-drive as before. A decision that lands while this holds the lease is dropped (it
11612
+ * cannot take the lease), whichever way this pass goes: with a run it dead-letters, or with one it
11613
+ * graces or leaves waiting on a `queued` task, whose next lapse then counts the turn and runs it
11614
+ * again. A narrow race, accepted.
11615
+ *
11616
+ * The marker grows by up to 11 characters (`:grace:lost`) beyond the dispatched id, which is
11617
+ * `<runId>:wf:<8 hex>`: `runId` plus 23 in all. A store that bounds the column (TypeORM's
11618
+ * `varchar(191)`) therefore dispatches a run whose id is 169 to 179 characters long, then refuses
11619
+ * the grace or lost write for it.
11620
+ */
11621
+ async noteLapsedTurn(runId, nowMs, seen) {
11622
+ const run = await this.store.getRun(runId);
11623
+ const awaited = run?.awaitingDecisionTaskId;
11624
+ if (seen) seen.marker = awaited;
11625
+ if (!run || run.status !== "suspended" || awaited === void 0 || awaited.endsWith(LOST_TURN_SUFFIX) || run.wakeAt == null || run.wakeAt > nowMs) {
11626
+ return void 0;
11627
+ }
11628
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) return void 0;
11629
+ const windowMs = this.dispatchedTurnSilenceMs;
11630
+ if (windowMs != null && nowMs < this.bootedAtMs + windowMs && !awaited.endsWith(GRACED_TURN_SUFFIX)) {
11631
+ return this.leaveTurnWaiting(run, this.bootedAtMs + windowMs, {
11632
+ awaitingDecisionTaskId: `${awaited}${GRACED_TURN_SUFFIX}`
11633
+ });
11634
+ }
11635
+ const state = await this.dispatchedTurnState(run, awaitedTaskId(awaited));
11636
+ if (state === "queued") return this.leaveTurnWaiting(run, nowMs + (windowMs ?? this.leaseMs));
11637
+ if (state === "error") return void 0;
11638
+ return this.countRecovery(run, {
11639
+ awaitingDecisionTaskId: `${awaited}${LOST_TURN_SUFFIX}`
11640
+ });
11641
+ }
11642
+ /** Where the broker has the task of a dispatched turn: see {@link WorkflowExecutor.taskState}. */
11643
+ async dispatchedTurnState(run, taskId) {
11644
+ const executor = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.remote?.executor;
11645
+ if (!executor?.taskState) return "unknown";
11646
+ try {
11647
+ return await executor.taskState(run, taskId);
11648
+ } catch {
11649
+ return "error";
11650
+ }
11651
+ }
11652
+ /** Push a lapsed turn's deadline to `until` without re-driving it, and give the lease back. */
11653
+ async leaveTurnWaiting(run, until, extra = {}) {
11654
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11655
+ "suspended"
11656
+ ], {
11657
+ ...extra,
11658
+ wakeAt: until,
11659
+ updatedAt: /* @__PURE__ */ new Date()
11660
+ });
11661
+ await this.store.releaseRunLock(run.id);
11662
+ return {
11663
+ runId: run.id,
11664
+ status: "suspended"
11665
+ };
11528
11666
  }
11529
11667
  /**
11530
11668
  * The reconcile safety net's per-run check (see {@link resumeDueTimers}): if this suspended run has
@@ -13826,6 +13964,12 @@ ${run.workflowVersion}`;
13826
13964
  const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
13827
13965
  status: "suspended",
13828
13966
  wakeAt: this.reconcileWakeAt(outcome.wakeAt),
13967
+ // A turn that ran here suspended the run, so no dispatched turn is awaited any more: a marker
13968
+ // left by an earlier dispatch (a re-drive that followed an onError handoff, say) would make
13969
+ // this suspend read as a lost turn at its next reconcile wake.
13970
+ ...run.awaitingDecisionTaskId !== void 0 ? {
13971
+ awaitingDecisionTaskId: void 0
13972
+ } : {},
13829
13973
  updatedAt: stamp
13830
13974
  }));
13831
13975
  if (!applied) return this.echoCurrentStatus(run.id);
@@ -14359,6 +14503,7 @@ ${run.workflowVersion}`;
14359
14503
  };
14360
14504
  const events = [];
14361
14505
  for (const cp of checkpoints) {
14506
+ if (isRequeuedStepFailure(cp)) continue;
14362
14507
  if (cp.status === "completed" || cp.status === "failed") {
14363
14508
  if (cp.seq >= HANDOFF_SEQ_BASE) continue;
14364
14509
  if (isAnyCheckpoint(cp)) {
@@ -14443,7 +14588,7 @@ ${run.workflowVersion}`;
14443
14588
  });
14444
14589
  } else if (cmd.kind === "call") {
14445
14590
  const existing = await this.store.getCheckpoint(run.id, cmd.seq);
14446
- if (existing) {
14591
+ if (existing && !isRequeuedStepFailure(existing)) {
14447
14592
  const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
14448
14593
  const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
14449
14594
  if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
@@ -16648,7 +16793,7 @@ ${run.workflowVersion}`;
16648
16793
  */
16649
16794
  async completeRemoteDecision(decision) {
16650
16795
  const run = await this.store.getRun(decision.runId);
16651
- if (!run || run.awaitingDecisionTaskId !== decision.taskId) return;
16796
+ if (!run || !awaitsDecision(run, decision)) return;
16652
16797
  if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
16653
16798
  if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
16654
16799
  return;
@@ -16660,16 +16805,20 @@ ${run.workflowVersion}`;
16660
16805
  const held = this.holdExecutionLease(decision.runId);
16661
16806
  try {
16662
16807
  const fresh = await this.store.getRun(decision.runId);
16663
- if (!fresh || fresh.awaitingDecisionTaskId !== decision.taskId) return;
16808
+ if (!fresh || !awaitsDecision(fresh, decision)) return;
16664
16809
  if (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead") {
16665
16810
  return;
16666
16811
  }
16667
16812
  await this.store.updateRun(decision.runId, {
16668
16813
  awaitingDecisionTaskId: void 0,
16669
16814
  wakeAt: void 0,
16815
+ ...fresh.recoveryAttempts ? {
16816
+ recoveryAttempts: 0
16817
+ } : {},
16670
16818
  updatedAt: /* @__PURE__ */ new Date()
16671
16819
  });
16672
16820
  fresh.awaitingDecisionTaskId = void 0;
16821
+ if (fresh.recoveryAttempts) fresh.recoveryAttempts = 0;
16673
16822
  await this.applyDecision(fresh, decision);
16674
16823
  } finally {
16675
16824
  const fenced = held.lease.lost;
@@ -16705,6 +16854,9 @@ ${run.workflowVersion}`;
16705
16854
  this.decisionDeadlineRearms.delete(runId);
16706
16855
  return;
16707
16856
  }
16857
+ if (run.awaitingDecisionTaskId.endsWith(GRACED_TURN_SUFFIX) && !await this.clearTurnGrace(runId, run.awaitingDecisionTaskId, now)) {
16858
+ return;
16859
+ }
16708
16860
  const halfWindow = Math.ceil(silenceMs / 2);
16709
16861
  if (run.wakeAt != null && run.wakeAt > now + halfWindow) {
16710
16862
  this.decisionDeadlineRearms.set(runId, run.wakeAt - halfWindow);
@@ -16721,6 +16873,29 @@ ${run.workflowVersion}`;
16721
16873
  }
16722
16874
  }
16723
16875
  /**
16876
+ * Write a graced turn's marker back to its bare task id (see {@link GRACED_TURN_SUFFIX}), under the
16877
+ * run's lease and only while the marker is still `marker`: a decision and a re-dispatch both write
16878
+ * the marker under that lease, so this never puts back the id of a turn they already replaced.
16879
+ * False when the lease was busy or the write failed; the caller's next beat tries again.
16880
+ */
16881
+ async clearTurnGrace(runId, marker, nowMs) {
16882
+ if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
16883
+ return false;
16884
+ }
16885
+ try {
16886
+ const fresh = await this.store.getRun(runId);
16887
+ if (fresh?.awaitingDecisionTaskId === marker) {
16888
+ await this.store.updateRun(runId, {
16889
+ awaitingDecisionTaskId: marker.slice(0, -GRACED_TURN_SUFFIX.length),
16890
+ updatedAt: /* @__PURE__ */ new Date()
16891
+ });
16892
+ }
16893
+ return true;
16894
+ } finally {
16895
+ await this.store.releaseRunLock(runId);
16896
+ }
16897
+ }
16898
+ /**
16724
16899
  * A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they
16725
16900
  * re-contend now instead of at their retry tick. Snapshot-and-clear — a run still blocked after the
16726
16901
  * retry re-registers itself, and one that's gone (cancelled/admitted) is dropped. Best-effort.
@@ -19116,6 +19291,7 @@ var InMemoryRunQueue = class {
19116
19291
  const waiting = lane.queued.get(run.runId);
19117
19292
  if (waiting) {
19118
19293
  if (run.reason === "wake") waiting.reason = "wake";
19294
+ if (waiting.awaiting !== run.awaiting) waiting.awaiting = void 0;
19119
19295
  continue;
19120
19296
  }
19121
19297
  const entry = {