@dudousxd/nestjs-durable-core 0.105.0 → 0.106.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -3193,6 +3193,11 @@ var RemoteWorkflowExecutor = class {
3193
3193
  };
3194
3194
  await this.transport.dispatchWorkflowTask(task);
3195
3195
  }
3196
+ /** Where the broker has the task dispatched under `taskId`, when the transport can tell. */
3197
+ async taskState(_run, taskId) {
3198
+ if (!this.transport.workflowTaskState) return "unknown";
3199
+ return this.transport.workflowTaskState(tenantGroup(sanitizeQueueToken(this.name), this.partition), taskId);
3200
+ }
3196
3201
  };
3197
3202
 
3198
3203
  // src/run-dispatch-batcher.ts
@@ -7013,6 +7018,20 @@ function mergeTags(staticTags, runTags) {
7013
7018
  }
7014
7019
  __name(mergeTags, "mergeTags");
7015
7020
  var BREAKPOINT = "breakpoint";
7021
+ var LOST_TURN_SUFFIX = ":lost";
7022
+ var GRACED_TURN_SUFFIX = ":grace";
7023
+ function awaitsDecision(run, decision) {
7024
+ const marker = run.awaitingDecisionTaskId;
7025
+ return marker !== void 0 && awaitedTaskId(marker) === decision.taskId;
7026
+ }
7027
+ __name(awaitsDecision, "awaitsDecision");
7028
+ function awaitedTaskId(marker) {
7029
+ let id = marker;
7030
+ if (id.endsWith(LOST_TURN_SUFFIX)) id = id.slice(0, -LOST_TURN_SUFFIX.length);
7031
+ if (id.endsWith(GRACED_TURN_SUFFIX)) id = id.slice(0, -GRACED_TURN_SUFFIX.length);
7032
+ return id;
7033
+ }
7034
+ __name(awaitedTaskId, "awaitedTaskId");
7016
7035
  var PENDING_POLL_BATCH = 100;
7017
7036
  var DEFAULT_STICKY_RUNS = 1e3;
7018
7037
  var DEFAULT_STICKY_IDLE_MS = 3e5;
@@ -7282,6 +7301,9 @@ var WorkflowEngine = class _WorkflowEngine {
7282
7301
  * `clock() + leaseMs`, so at or after it). See {@link orphanedRuns}.
7283
7302
  */
7284
7303
  ownLeaseCutoffMs;
7304
+ /** When this engine was built (clock ms): {@link noteLapsedTurn} counts no lost turn within one
7305
+ * turn window of it, while live workers' beats catch up with deadlines that went stale. */
7306
+ bootedAtMs;
7285
7307
  maxRecoveryAttempts;
7286
7308
  unregisteredWorkflowGraceMs;
7287
7309
  remoteAdvanceSilenceMs;
@@ -7511,6 +7533,7 @@ var WorkflowEngine = class _WorkflowEngine {
7511
7533
  this.namespace = poolNamespace(deps.namespace);
7512
7534
  this.leaseMs = deps.leaseMs ?? 3e4;
7513
7535
  this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
7536
+ this.bootedAtMs = this.clock();
7514
7537
  this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
7515
7538
  const grace = deps.unregisteredWorkflowGraceMs;
7516
7539
  this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
@@ -7888,6 +7911,9 @@ var WorkflowEngine = class _WorkflowEngine {
7888
7911
  return;
7889
7912
  }
7890
7913
  if (dispatch.reason !== "wake" || TERMINAL_RUN_STATUSES.includes(run.status)) return;
7914
+ if (dispatch.awaiting !== void 0 && (run.awaitingDecisionTaskId === void 0 || awaitedTaskId(run.awaitingDecisionTaskId) !== dispatch.awaiting)) {
7915
+ return;
7916
+ }
7891
7917
  const nowMs = this.clock();
7892
7918
  if (run.wakeAt !== void 0 && run.wakeAt <= nowMs) {
7893
7919
  const waiters = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -7939,6 +7965,7 @@ var WorkflowEngine = class _WorkflowEngine {
7939
7965
  */
7940
7966
  async handOffDue(runs, nowMs, settled) {
7941
7967
  const claimed = [];
7968
+ const awaiting = /* @__PURE__ */ new Map();
7942
7969
  for (const run of runs) {
7943
7970
  if (this.draining) break;
7944
7971
  this.markClaim(run.id);
@@ -7953,16 +7980,26 @@ var WorkflowEngine = class _WorkflowEngine {
7953
7980
  continue;
7954
7981
  }
7955
7982
  try {
7983
+ const seen = {};
7984
+ const handled = await this.noteLapsedTurn(run.id, nowMs, seen);
7985
+ if (handled) {
7986
+ if (handled.status === "dead") settled?.set(run.id, handled);
7987
+ continue;
7988
+ }
7956
7989
  const patch = {
7957
7990
  wakeAt: nowMs + this.leaseMs
7958
7991
  };
7992
+ let won = true;
7959
7993
  if (this.store.updateRunIf) {
7960
- if (await this.store.updateRunIf(run.id, [
7994
+ won = await this.store.updateRunIf(run.id, [
7961
7995
  run.status
7962
- ], patch)) claimed.push(run);
7996
+ ], patch);
7963
7997
  } else {
7964
7998
  await this.store.updateRun(run.id, patch);
7999
+ }
8000
+ if (won) {
7965
8001
  claimed.push(run);
8002
+ if (seen.marker !== void 0) awaiting.set(run.id, awaitedTaskId(seen.marker));
7966
8003
  }
7967
8004
  } finally {
7968
8005
  await this.store.releaseRunLock(run.id, this.instanceId);
@@ -7973,13 +8010,19 @@ var WorkflowEngine = class _WorkflowEngine {
7973
8010
  }
7974
8011
  try {
7975
8012
  for (let at = 0; at < claimed.length; at += RUN_DISPATCH_BATCH) {
7976
- await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => ({
7977
- runId: run.id,
7978
- reason: "wake",
7979
- ...run.priority !== void 0 ? {
7980
- priority: run.priority
7981
- } : {}
7982
- })), {
8013
+ await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => {
8014
+ const turn = awaiting.get(run.id);
8015
+ return {
8016
+ runId: run.id,
8017
+ reason: "wake",
8018
+ ...run.priority !== void 0 ? {
8019
+ priority: run.priority
8020
+ } : {},
8021
+ ...turn !== void 0 ? {
8022
+ awaiting: turn
8023
+ } : {}
8024
+ };
8025
+ }), {
7983
8026
  namespace: this.namespace
7984
8027
  });
7985
8028
  }
@@ -11053,28 +11096,39 @@ var WorkflowEngine = class _WorkflowEngine {
11053
11096
  * `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
11054
11097
  * result to skip the resume, or `undefined` to proceed.
11055
11098
  */
11056
- async countRecovery(run) {
11099
+ async countRecovery(run, extra = {}) {
11057
11100
  const attempts = (run.recoveryAttempts ?? 0) + 1;
11101
+ const updateIf = /* @__PURE__ */ __name((patch) => (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11102
+ run.status
11103
+ ], patch), "updateIf");
11104
+ const moved = /* @__PURE__ */ __name(async () => {
11105
+ await this.store.releaseRunLock(run.id);
11106
+ return this.echoCurrentStatus(run.id);
11107
+ }, "moved");
11058
11108
  if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
11059
11109
  const error2 = {
11060
11110
  message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
11061
11111
  code: "max_recovery_attempts"
11062
11112
  };
11063
- await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
11113
+ const written = await updateIf(await this.withLifecycleMarker(run, {
11064
11114
  status: "dead",
11065
11115
  error: error2,
11066
11116
  updatedAt: /* @__PURE__ */ new Date()
11067
11117
  }));
11118
+ if (!written) return moved();
11068
11119
  await this.store.releaseRunLock(run.id);
11069
11120
  return this.announceDead({
11070
11121
  ...run,
11071
11122
  recoveryAttempts: attempts
11072
11123
  }, error2);
11073
11124
  }
11074
- await this.store.updateRun(run.id, {
11125
+ if (!await updateIf({
11126
+ ...extra,
11075
11127
  recoveryAttempts: attempts,
11076
11128
  updatedAt: /* @__PURE__ */ new Date()
11077
- });
11129
+ })) {
11130
+ return moved();
11131
+ }
11078
11132
  return void 0;
11079
11133
  }
11080
11134
  /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
@@ -11221,7 +11275,87 @@ var WorkflowEngine = class _WorkflowEngine {
11221
11275
  return this.resumeLeased([
11222
11276
  ...due,
11223
11277
  ...blocked
11224
- ], nowMs, (run) => this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11278
+ ], nowMs, async (run) => await this.noteLapsedTurn(run.id, nowMs) ?? this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11279
+ }
11280
+ /**
11281
+ * Called under the run's lease by the due-timer pass, for a run that THIS instance would re-drive.
11282
+ * A run suspended awaiting a DISPATCHED workflow turn's decision (`awaitingDecisionTaskId`) whose
11283
+ * re-drive deadline (`wakeAt`) has passed lost that turn: no decision came back and the worker
11284
+ * stopped beating (every beat pushes `wakeAt` forward, see {@link rearmDecisionDeadline}), so the
11285
+ * worker running it died, or the task was lost with it. That is the dispatched form of a crashed
11286
+ * inline turn, so it spends the same {@link maxRecoveryAttempts} budget through
11287
+ * {@link countRecovery}: a turn that keeps killing its worker ends `dead` (with the same
11288
+ * `run.failed` announcement) instead of being re-dispatched, and redelivered, for ever.
11289
+ *
11290
+ * Counted at most once per awaited turn: the count rewrites the marker to its `:lost` form, which
11291
+ * no later pass counts again however long the re-drive waits on the run queue, and the re-drive's
11292
+ * dispatch replaces it with the next turn's id. Not counted, and the re-drive left for later:
11293
+ * - while the broker still holds the task unstarted (a backed-up queue, not a dead worker), or
11294
+ * - within one turn window of this engine's boot, when beats from live workers have not yet had
11295
+ * the chance to push a deadline that went stale while the engine was down. Once per turn (the
11296
+ * `:grace` marker): a lapse after that takes the normal path, whichever instance judges it.
11297
+ * A late decision of a counted turn is still applied while the re-drive has not dispatched yet
11298
+ * (see `awaitsDecision`), and gives the attempt back.
11299
+ * Never counted by an instance that cannot resolve the workflow (it re-drives nothing), nor for a
11300
+ * decision that arrives, a worker that keeps beating, or any write a healthy turn makes on its way.
11301
+ *
11302
+ * Re-reads under the lease: the listing may predate a decision applied since. Returns a result when
11303
+ * the run was dead-lettered or left waiting (the lease released either way), else `undefined` for
11304
+ * the caller to re-drive as before. A decision that lands while this holds the lease is dropped (it
11305
+ * cannot take the lease), whichever way this pass goes: with a run it dead-letters, or with one it
11306
+ * graces or leaves waiting on a `queued` task, whose next lapse then counts the turn and runs it
11307
+ * again. A narrow race, accepted.
11308
+ *
11309
+ * The marker grows by up to 11 characters (`:grace:lost`) beyond the dispatched id, which is
11310
+ * `<runId>:wf:<8 hex>`: `runId` plus 23 in all. A store that bounds the column (TypeORM's
11311
+ * `varchar(191)`) therefore dispatches a run whose id is 169 to 179 characters long, then refuses
11312
+ * the grace or lost write for it.
11313
+ */
11314
+ async noteLapsedTurn(runId, nowMs, seen) {
11315
+ const run = await this.store.getRun(runId);
11316
+ const awaited = run?.awaitingDecisionTaskId;
11317
+ if (seen) seen.marker = awaited;
11318
+ if (!run || run.status !== "suspended" || awaited === void 0 || awaited.endsWith(LOST_TURN_SUFFIX) || run.wakeAt == null || run.wakeAt > nowMs) {
11319
+ return void 0;
11320
+ }
11321
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) return void 0;
11322
+ const windowMs = this.dispatchedTurnSilenceMs;
11323
+ if (windowMs != null && nowMs < this.bootedAtMs + windowMs && !awaited.endsWith(GRACED_TURN_SUFFIX)) {
11324
+ return this.leaveTurnWaiting(run, this.bootedAtMs + windowMs, {
11325
+ awaitingDecisionTaskId: `${awaited}${GRACED_TURN_SUFFIX}`
11326
+ });
11327
+ }
11328
+ const state = await this.dispatchedTurnState(run, awaitedTaskId(awaited));
11329
+ if (state === "queued") return this.leaveTurnWaiting(run, nowMs + (windowMs ?? this.leaseMs));
11330
+ if (state === "error") return void 0;
11331
+ return this.countRecovery(run, {
11332
+ awaitingDecisionTaskId: `${awaited}${LOST_TURN_SUFFIX}`
11333
+ });
11334
+ }
11335
+ /** Where the broker has the task of a dispatched turn: see {@link WorkflowExecutor.taskState}. */
11336
+ async dispatchedTurnState(run, taskId) {
11337
+ const executor = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.remote?.executor;
11338
+ if (!executor?.taskState) return "unknown";
11339
+ try {
11340
+ return await executor.taskState(run, taskId);
11341
+ } catch {
11342
+ return "error";
11343
+ }
11344
+ }
11345
+ /** Push a lapsed turn's deadline to `until` without re-driving it, and give the lease back. */
11346
+ async leaveTurnWaiting(run, until, extra = {}) {
11347
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11348
+ "suspended"
11349
+ ], {
11350
+ ...extra,
11351
+ wakeAt: until,
11352
+ updatedAt: /* @__PURE__ */ new Date()
11353
+ });
11354
+ await this.store.releaseRunLock(run.id);
11355
+ return {
11356
+ runId: run.id,
11357
+ status: "suspended"
11358
+ };
11225
11359
  }
11226
11360
  /**
11227
11361
  * The reconcile safety net's per-run check (see {@link resumeDueTimers}): if this suspended run has
@@ -13523,6 +13657,12 @@ ${run.workflowVersion}`;
13523
13657
  const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
13524
13658
  status: "suspended",
13525
13659
  wakeAt: this.reconcileWakeAt(outcome.wakeAt),
13660
+ // A turn that ran here suspended the run, so no dispatched turn is awaited any more: a marker
13661
+ // left by an earlier dispatch (a re-drive that followed an onError handoff, say) would make
13662
+ // this suspend read as a lost turn at its next reconcile wake.
13663
+ ...run.awaitingDecisionTaskId !== void 0 ? {
13664
+ awaitingDecisionTaskId: void 0
13665
+ } : {},
13526
13666
  updatedAt: stamp
13527
13667
  }));
13528
13668
  if (!applied) return this.echoCurrentStatus(run.id);
@@ -16345,7 +16485,7 @@ ${run.workflowVersion}`;
16345
16485
  */
16346
16486
  async completeRemoteDecision(decision) {
16347
16487
  const run = await this.store.getRun(decision.runId);
16348
- if (!run || run.awaitingDecisionTaskId !== decision.taskId) return;
16488
+ if (!run || !awaitsDecision(run, decision)) return;
16349
16489
  if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
16350
16490
  if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
16351
16491
  return;
@@ -16357,16 +16497,20 @@ ${run.workflowVersion}`;
16357
16497
  const held = this.holdExecutionLease(decision.runId);
16358
16498
  try {
16359
16499
  const fresh = await this.store.getRun(decision.runId);
16360
- if (!fresh || fresh.awaitingDecisionTaskId !== decision.taskId) return;
16500
+ if (!fresh || !awaitsDecision(fresh, decision)) return;
16361
16501
  if (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead") {
16362
16502
  return;
16363
16503
  }
16364
16504
  await this.store.updateRun(decision.runId, {
16365
16505
  awaitingDecisionTaskId: void 0,
16366
16506
  wakeAt: void 0,
16507
+ ...fresh.recoveryAttempts ? {
16508
+ recoveryAttempts: 0
16509
+ } : {},
16367
16510
  updatedAt: /* @__PURE__ */ new Date()
16368
16511
  });
16369
16512
  fresh.awaitingDecisionTaskId = void 0;
16513
+ if (fresh.recoveryAttempts) fresh.recoveryAttempts = 0;
16370
16514
  await this.applyDecision(fresh, decision);
16371
16515
  } finally {
16372
16516
  const fenced = held.lease.lost;
@@ -16402,6 +16546,9 @@ ${run.workflowVersion}`;
16402
16546
  this.decisionDeadlineRearms.delete(runId);
16403
16547
  return;
16404
16548
  }
16549
+ if (run.awaitingDecisionTaskId.endsWith(GRACED_TURN_SUFFIX) && !await this.clearTurnGrace(runId, run.awaitingDecisionTaskId, now)) {
16550
+ return;
16551
+ }
16405
16552
  const halfWindow = Math.ceil(silenceMs / 2);
16406
16553
  if (run.wakeAt != null && run.wakeAt > now + halfWindow) {
16407
16554
  this.decisionDeadlineRearms.set(runId, run.wakeAt - halfWindow);
@@ -16418,6 +16565,29 @@ ${run.workflowVersion}`;
16418
16565
  }
16419
16566
  }
16420
16567
  /**
16568
+ * Write a graced turn's marker back to its bare task id (see {@link GRACED_TURN_SUFFIX}), under the
16569
+ * run's lease and only while the marker is still `marker`: a decision and a re-dispatch both write
16570
+ * the marker under that lease, so this never puts back the id of a turn they already replaced.
16571
+ * False when the lease was busy or the write failed; the caller's next beat tries again.
16572
+ */
16573
+ async clearTurnGrace(runId, marker, nowMs) {
16574
+ if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
16575
+ return false;
16576
+ }
16577
+ try {
16578
+ const fresh = await this.store.getRun(runId);
16579
+ if (fresh?.awaitingDecisionTaskId === marker) {
16580
+ await this.store.updateRun(runId, {
16581
+ awaitingDecisionTaskId: marker.slice(0, -GRACED_TURN_SUFFIX.length),
16582
+ updatedAt: /* @__PURE__ */ new Date()
16583
+ });
16584
+ }
16585
+ return true;
16586
+ } finally {
16587
+ await this.store.releaseRunLock(runId);
16588
+ }
16589
+ }
16590
+ /**
16421
16591
  * A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they
16422
16592
  * re-contend now instead of at their retry tick. Snapshot-and-clear — a run still blocked after the
16423
16593
  * retry re-registers itself, and one that's gone (cancelled/admitted) is dropped. Best-effort.
@@ -18813,6 +18983,7 @@ var InMemoryRunQueue = class {
18813
18983
  const waiting = lane.queued.get(run.runId);
18814
18984
  if (waiting) {
18815
18985
  if (run.reason === "wake") waiting.reason = "wake";
18986
+ if (waiting.awaiting !== run.awaiting) waiting.awaiting = void 0;
18816
18987
  continue;
18817
18988
  }
18818
18989
  const entry = {