@dudousxd/nestjs-durable-core 0.105.0 → 0.106.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -3193,6 +3193,11 @@ var RemoteWorkflowExecutor = class {
3193
3193
  };
3194
3194
  await this.transport.dispatchWorkflowTask(task);
3195
3195
  }
3196
+ /** Where the broker has the task dispatched under `taskId`, when the transport can tell. */
3197
+ async taskState(_run, taskId) {
3198
+ if (!this.transport.workflowTaskState) return "unknown";
3199
+ return this.transport.workflowTaskState(tenantGroup(sanitizeQueueToken(this.name), this.partition), taskId);
3200
+ }
3196
3201
  };
3197
3202
 
3198
3203
  // src/run-dispatch-batcher.ts
@@ -7002,6 +7007,10 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
7002
7007
  __name(createWorkflowCtx, "createWorkflowCtx");
7003
7008
 
7004
7009
  // src/engine.ts
7010
+ function isRequeuedStepFailure(cp) {
7011
+ return cp.status === "failed" && cp.attempts === 0 && cp.seq >= 0 && (cp.kind === "local" || cp.kind === "remote");
7012
+ }
7013
+ __name(isRequeuedStepFailure, "isRequeuedStepFailure");
7005
7014
  function mergeTags(staticTags, runTags) {
7006
7015
  if (!staticTags?.length && !runTags?.length) return void 0;
7007
7016
  return [
@@ -7013,6 +7022,20 @@ function mergeTags(staticTags, runTags) {
7013
7022
  }
7014
7023
  __name(mergeTags, "mergeTags");
7015
7024
  var BREAKPOINT = "breakpoint";
7025
+ var LOST_TURN_SUFFIX = ":lost";
7026
+ var GRACED_TURN_SUFFIX = ":grace";
7027
+ function awaitsDecision(run, decision) {
7028
+ const marker = run.awaitingDecisionTaskId;
7029
+ return marker !== void 0 && awaitedTaskId(marker) === decision.taskId;
7030
+ }
7031
+ __name(awaitsDecision, "awaitsDecision");
7032
+ function awaitedTaskId(marker) {
7033
+ let id = marker;
7034
+ if (id.endsWith(LOST_TURN_SUFFIX)) id = id.slice(0, -LOST_TURN_SUFFIX.length);
7035
+ if (id.endsWith(GRACED_TURN_SUFFIX)) id = id.slice(0, -GRACED_TURN_SUFFIX.length);
7036
+ return id;
7037
+ }
7038
+ __name(awaitedTaskId, "awaitedTaskId");
7016
7039
  var PENDING_POLL_BATCH = 100;
7017
7040
  var DEFAULT_STICKY_RUNS = 1e3;
7018
7041
  var DEFAULT_STICKY_IDLE_MS = 3e5;
@@ -7282,6 +7305,9 @@ var WorkflowEngine = class _WorkflowEngine {
7282
7305
  * `clock() + leaseMs`, so at or after it). See {@link orphanedRuns}.
7283
7306
  */
7284
7307
  ownLeaseCutoffMs;
7308
+ /** When this engine was built (clock ms): {@link noteLapsedTurn} counts no lost turn within one
7309
+ * turn window of it, while live workers' beats catch up with deadlines that went stale. */
7310
+ bootedAtMs;
7285
7311
  maxRecoveryAttempts;
7286
7312
  unregisteredWorkflowGraceMs;
7287
7313
  remoteAdvanceSilenceMs;
@@ -7511,6 +7537,7 @@ var WorkflowEngine = class _WorkflowEngine {
7511
7537
  this.namespace = poolNamespace(deps.namespace);
7512
7538
  this.leaseMs = deps.leaseMs ?? 3e4;
7513
7539
  this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
7540
+ this.bootedAtMs = this.clock();
7514
7541
  this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
7515
7542
  const grace = deps.unregisteredWorkflowGraceMs;
7516
7543
  this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
@@ -7888,6 +7915,9 @@ var WorkflowEngine = class _WorkflowEngine {
7888
7915
  return;
7889
7916
  }
7890
7917
  if (dispatch.reason !== "wake" || TERMINAL_RUN_STATUSES.includes(run.status)) return;
7918
+ if (dispatch.awaiting !== void 0 && (run.awaitingDecisionTaskId === void 0 || awaitedTaskId(run.awaitingDecisionTaskId) !== dispatch.awaiting)) {
7919
+ return;
7920
+ }
7891
7921
  const nowMs = this.clock();
7892
7922
  if (run.wakeAt !== void 0 && run.wakeAt <= nowMs) {
7893
7923
  const waiters = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -7939,6 +7969,7 @@ var WorkflowEngine = class _WorkflowEngine {
7939
7969
  */
7940
7970
  async handOffDue(runs, nowMs, settled) {
7941
7971
  const claimed = [];
7972
+ const awaiting = /* @__PURE__ */ new Map();
7942
7973
  for (const run of runs) {
7943
7974
  if (this.draining) break;
7944
7975
  this.markClaim(run.id);
@@ -7953,16 +7984,26 @@ var WorkflowEngine = class _WorkflowEngine {
7953
7984
  continue;
7954
7985
  }
7955
7986
  try {
7987
+ const seen = {};
7988
+ const handled = await this.noteLapsedTurn(run.id, nowMs, seen);
7989
+ if (handled) {
7990
+ if (handled.status === "dead") settled?.set(run.id, handled);
7991
+ continue;
7992
+ }
7956
7993
  const patch = {
7957
7994
  wakeAt: nowMs + this.leaseMs
7958
7995
  };
7996
+ let won = true;
7959
7997
  if (this.store.updateRunIf) {
7960
- if (await this.store.updateRunIf(run.id, [
7998
+ won = await this.store.updateRunIf(run.id, [
7961
7999
  run.status
7962
- ], patch)) claimed.push(run);
8000
+ ], patch);
7963
8001
  } else {
7964
8002
  await this.store.updateRun(run.id, patch);
8003
+ }
8004
+ if (won) {
7965
8005
  claimed.push(run);
8006
+ if (seen.marker !== void 0) awaiting.set(run.id, awaitedTaskId(seen.marker));
7966
8007
  }
7967
8008
  } finally {
7968
8009
  await this.store.releaseRunLock(run.id, this.instanceId);
@@ -7973,13 +8014,19 @@ var WorkflowEngine = class _WorkflowEngine {
7973
8014
  }
7974
8015
  try {
7975
8016
  for (let at = 0; at < claimed.length; at += RUN_DISPATCH_BATCH) {
7976
- await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => ({
7977
- runId: run.id,
7978
- reason: "wake",
7979
- ...run.priority !== void 0 ? {
7980
- priority: run.priority
7981
- } : {}
7982
- })), {
8017
+ await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => {
8018
+ const turn = awaiting.get(run.id);
8019
+ return {
8020
+ runId: run.id,
8021
+ reason: "wake",
8022
+ ...run.priority !== void 0 ? {
8023
+ priority: run.priority
8024
+ } : {},
8025
+ ...turn !== void 0 ? {
8026
+ awaiting: turn
8027
+ } : {}
8028
+ };
8029
+ }), {
7983
8030
  namespace: this.namespace
7984
8031
  });
7985
8032
  }
@@ -11053,28 +11100,39 @@ var WorkflowEngine = class _WorkflowEngine {
11053
11100
  * `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
11054
11101
  * result to skip the resume, or `undefined` to proceed.
11055
11102
  */
11056
- async countRecovery(run) {
11103
+ async countRecovery(run, extra = {}) {
11057
11104
  const attempts = (run.recoveryAttempts ?? 0) + 1;
11105
+ const updateIf = /* @__PURE__ */ __name((patch) => (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11106
+ run.status
11107
+ ], patch), "updateIf");
11108
+ const moved = /* @__PURE__ */ __name(async () => {
11109
+ await this.store.releaseRunLock(run.id);
11110
+ return this.echoCurrentStatus(run.id);
11111
+ }, "moved");
11058
11112
  if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
11059
11113
  const error2 = {
11060
11114
  message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
11061
11115
  code: "max_recovery_attempts"
11062
11116
  };
11063
- await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
11117
+ const written = await updateIf(await this.withLifecycleMarker(run, {
11064
11118
  status: "dead",
11065
11119
  error: error2,
11066
11120
  updatedAt: /* @__PURE__ */ new Date()
11067
11121
  }));
11122
+ if (!written) return moved();
11068
11123
  await this.store.releaseRunLock(run.id);
11069
11124
  return this.announceDead({
11070
11125
  ...run,
11071
11126
  recoveryAttempts: attempts
11072
11127
  }, error2);
11073
11128
  }
11074
- await this.store.updateRun(run.id, {
11129
+ if (!await updateIf({
11130
+ ...extra,
11075
11131
  recoveryAttempts: attempts,
11076
11132
  updatedAt: /* @__PURE__ */ new Date()
11077
- });
11133
+ })) {
11134
+ return moved();
11135
+ }
11078
11136
  return void 0;
11079
11137
  }
11080
11138
  /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
@@ -11221,7 +11279,87 @@ var WorkflowEngine = class _WorkflowEngine {
11221
11279
  return this.resumeLeased([
11222
11280
  ...due,
11223
11281
  ...blocked
11224
- ], nowMs, (run) => this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11282
+ ], nowMs, async (run) => await this.noteLapsedTurn(run.id, nowMs) ?? this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11283
+ }
11284
+ /**
11285
+ * Called under the run's lease by the due-timer pass, for a run that THIS instance would re-drive.
11286
+ * A run suspended awaiting a DISPATCHED workflow turn's decision (`awaitingDecisionTaskId`) whose
11287
+ * re-drive deadline (`wakeAt`) has passed lost that turn: no decision came back and the worker
11288
+ * stopped beating (every beat pushes `wakeAt` forward, see {@link rearmDecisionDeadline}), so the
11289
+ * worker running it died, or the task was lost with it. That is the dispatched form of a crashed
11290
+ * inline turn, so it spends the same {@link maxRecoveryAttempts} budget through
11291
+ * {@link countRecovery}: a turn that keeps killing its worker ends `dead` (with the same
11292
+ * `run.failed` announcement) instead of being re-dispatched, and redelivered, for ever.
11293
+ *
11294
+ * Counted at most once per awaited turn: the count rewrites the marker to its `:lost` form, which
11295
+ * no later pass counts again however long the re-drive waits on the run queue, and the re-drive's
11296
+ * dispatch replaces it with the next turn's id. Not counted, and the re-drive left for later:
11297
+ * - while the broker still holds the task unstarted (a backed-up queue, not a dead worker), or
11298
+ * - within one turn window of this engine's boot, when beats from live workers have not yet had
11299
+ * the chance to push a deadline that went stale while the engine was down. Once per turn (the
11300
+ * `:grace` marker): a lapse after that takes the normal path, whichever instance judges it.
11301
+ * A late decision of a counted turn is still applied while the re-drive has not dispatched yet
11302
+ * (see `awaitsDecision`), and gives the attempt back.
11303
+ * Never counted by an instance that cannot resolve the workflow (it re-drives nothing), nor for a
11304
+ * decision that arrives, a worker that keeps beating, or any write a healthy turn makes on its way.
11305
+ *
11306
+ * Re-reads under the lease: the listing may predate a decision applied since. Returns a result when
11307
+ * the run was dead-lettered or left waiting (the lease released either way), else `undefined` for
11308
+ * the caller to re-drive as before. A decision that lands while this holds the lease is dropped (it
11309
+ * cannot take the lease), whichever way this pass goes: with a run it dead-letters, or with one it
11310
+ * graces or leaves waiting on a `queued` task, whose next lapse then counts the turn and runs it
11311
+ * again. A narrow race, accepted.
11312
+ *
11313
+ * The marker grows by up to 11 characters (`:grace:lost`) beyond the dispatched id, which is
11314
+ * `<runId>:wf:<8 hex>`: `runId` plus 23 in all. A store that bounds the column (TypeORM's
11315
+ * `varchar(191)`) therefore dispatches a run whose id is 169 to 179 characters long, then refuses
11316
+ * the grace or lost write for it.
11317
+ */
11318
+ async noteLapsedTurn(runId, nowMs, seen) {
11319
+ const run = await this.store.getRun(runId);
11320
+ const awaited = run?.awaitingDecisionTaskId;
11321
+ if (seen) seen.marker = awaited;
11322
+ if (!run || run.status !== "suspended" || awaited === void 0 || awaited.endsWith(LOST_TURN_SUFFIX) || run.wakeAt == null || run.wakeAt > nowMs) {
11323
+ return void 0;
11324
+ }
11325
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) return void 0;
11326
+ const windowMs = this.dispatchedTurnSilenceMs;
11327
+ if (windowMs != null && nowMs < this.bootedAtMs + windowMs && !awaited.endsWith(GRACED_TURN_SUFFIX)) {
11328
+ return this.leaveTurnWaiting(run, this.bootedAtMs + windowMs, {
11329
+ awaitingDecisionTaskId: `${awaited}${GRACED_TURN_SUFFIX}`
11330
+ });
11331
+ }
11332
+ const state = await this.dispatchedTurnState(run, awaitedTaskId(awaited));
11333
+ if (state === "queued") return this.leaveTurnWaiting(run, nowMs + (windowMs ?? this.leaseMs));
11334
+ if (state === "error") return void 0;
11335
+ return this.countRecovery(run, {
11336
+ awaitingDecisionTaskId: `${awaited}${LOST_TURN_SUFFIX}`
11337
+ });
11338
+ }
11339
+ /** Where the broker has the task of a dispatched turn: see {@link WorkflowExecutor.taskState}. */
11340
+ async dispatchedTurnState(run, taskId) {
11341
+ const executor = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.remote?.executor;
11342
+ if (!executor?.taskState) return "unknown";
11343
+ try {
11344
+ return await executor.taskState(run, taskId);
11345
+ } catch {
11346
+ return "error";
11347
+ }
11348
+ }
11349
+ /** Push a lapsed turn's deadline to `until` without re-driving it, and give the lease back. */
11350
+ async leaveTurnWaiting(run, until, extra = {}) {
11351
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11352
+ "suspended"
11353
+ ], {
11354
+ ...extra,
11355
+ wakeAt: until,
11356
+ updatedAt: /* @__PURE__ */ new Date()
11357
+ });
11358
+ await this.store.releaseRunLock(run.id);
11359
+ return {
11360
+ runId: run.id,
11361
+ status: "suspended"
11362
+ };
11225
11363
  }
11226
11364
  /**
11227
11365
  * The reconcile safety net's per-run check (see {@link resumeDueTimers}): if this suspended run has
@@ -13523,6 +13661,12 @@ ${run.workflowVersion}`;
13523
13661
  const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
13524
13662
  status: "suspended",
13525
13663
  wakeAt: this.reconcileWakeAt(outcome.wakeAt),
13664
+ // A turn that ran here suspended the run, so no dispatched turn is awaited any more: a marker
13665
+ // left by an earlier dispatch (a re-drive that followed an onError handoff, say) would make
13666
+ // this suspend read as a lost turn at its next reconcile wake.
13667
+ ...run.awaitingDecisionTaskId !== void 0 ? {
13668
+ awaitingDecisionTaskId: void 0
13669
+ } : {},
13526
13670
  updatedAt: stamp
13527
13671
  }));
13528
13672
  if (!applied) return this.echoCurrentStatus(run.id);
@@ -14056,6 +14200,7 @@ ${run.workflowVersion}`;
14056
14200
  };
14057
14201
  const events = [];
14058
14202
  for (const cp of checkpoints) {
14203
+ if (isRequeuedStepFailure(cp)) continue;
14059
14204
  if (cp.status === "completed" || cp.status === "failed") {
14060
14205
  if (cp.seq >= HANDOFF_SEQ_BASE) continue;
14061
14206
  if (isAnyCheckpoint(cp)) {
@@ -14140,7 +14285,7 @@ ${run.workflowVersion}`;
14140
14285
  });
14141
14286
  } else if (cmd.kind === "call") {
14142
14287
  const existing = await this.store.getCheckpoint(run.id, cmd.seq);
14143
- if (existing) {
14288
+ if (existing && !isRequeuedStepFailure(existing)) {
14144
14289
  const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
14145
14290
  const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
14146
14291
  if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
@@ -16345,7 +16490,7 @@ ${run.workflowVersion}`;
16345
16490
  */
16346
16491
  async completeRemoteDecision(decision) {
16347
16492
  const run = await this.store.getRun(decision.runId);
16348
- if (!run || run.awaitingDecisionTaskId !== decision.taskId) return;
16493
+ if (!run || !awaitsDecision(run, decision)) return;
16349
16494
  if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
16350
16495
  if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
16351
16496
  return;
@@ -16357,16 +16502,20 @@ ${run.workflowVersion}`;
16357
16502
  const held = this.holdExecutionLease(decision.runId);
16358
16503
  try {
16359
16504
  const fresh = await this.store.getRun(decision.runId);
16360
- if (!fresh || fresh.awaitingDecisionTaskId !== decision.taskId) return;
16505
+ if (!fresh || !awaitsDecision(fresh, decision)) return;
16361
16506
  if (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead") {
16362
16507
  return;
16363
16508
  }
16364
16509
  await this.store.updateRun(decision.runId, {
16365
16510
  awaitingDecisionTaskId: void 0,
16366
16511
  wakeAt: void 0,
16512
+ ...fresh.recoveryAttempts ? {
16513
+ recoveryAttempts: 0
16514
+ } : {},
16367
16515
  updatedAt: /* @__PURE__ */ new Date()
16368
16516
  });
16369
16517
  fresh.awaitingDecisionTaskId = void 0;
16518
+ if (fresh.recoveryAttempts) fresh.recoveryAttempts = 0;
16370
16519
  await this.applyDecision(fresh, decision);
16371
16520
  } finally {
16372
16521
  const fenced = held.lease.lost;
@@ -16402,6 +16551,9 @@ ${run.workflowVersion}`;
16402
16551
  this.decisionDeadlineRearms.delete(runId);
16403
16552
  return;
16404
16553
  }
16554
+ if (run.awaitingDecisionTaskId.endsWith(GRACED_TURN_SUFFIX) && !await this.clearTurnGrace(runId, run.awaitingDecisionTaskId, now)) {
16555
+ return;
16556
+ }
16405
16557
  const halfWindow = Math.ceil(silenceMs / 2);
16406
16558
  if (run.wakeAt != null && run.wakeAt > now + halfWindow) {
16407
16559
  this.decisionDeadlineRearms.set(runId, run.wakeAt - halfWindow);
@@ -16418,6 +16570,29 @@ ${run.workflowVersion}`;
16418
16570
  }
16419
16571
  }
16420
16572
  /**
16573
+ * Write a graced turn's marker back to its bare task id (see {@link GRACED_TURN_SUFFIX}), under the
16574
+ * run's lease and only while the marker is still `marker`: a decision and a re-dispatch both write
16575
+ * the marker under that lease, so this never puts back the id of a turn they already replaced.
16576
+ * False when the lease was busy or the write failed; the caller's next beat tries again.
16577
+ */
16578
+ async clearTurnGrace(runId, marker, nowMs) {
16579
+ if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
16580
+ return false;
16581
+ }
16582
+ try {
16583
+ const fresh = await this.store.getRun(runId);
16584
+ if (fresh?.awaitingDecisionTaskId === marker) {
16585
+ await this.store.updateRun(runId, {
16586
+ awaitingDecisionTaskId: marker.slice(0, -GRACED_TURN_SUFFIX.length),
16587
+ updatedAt: /* @__PURE__ */ new Date()
16588
+ });
16589
+ }
16590
+ return true;
16591
+ } finally {
16592
+ await this.store.releaseRunLock(runId);
16593
+ }
16594
+ }
16595
+ /**
16421
16596
  * A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they
16422
16597
  * re-contend now instead of at their retry tick. Snapshot-and-clear — a run still blocked after the
16423
16598
  * retry re-registers itself, and one that's gone (cancelled/admitted) is dropped. Best-effort.
@@ -18813,6 +18988,7 @@ var InMemoryRunQueue = class {
18813
18988
  const waiting = lane.queued.get(run.runId);
18814
18989
  if (waiting) {
18815
18990
  if (run.reason === "wake") waiting.reason = "wake";
18991
+ if (waiting.awaiting !== run.awaiting) waiting.awaiting = void 0;
18816
18992
  continue;
18817
18993
  }
18818
18994
  const entry = {