@dudousxd/nestjs-durable-core 0.104.0 → 0.106.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -679,6 +679,27 @@ function backoffDelay(attempt, options) {
679
679
  return options?.jitter ? Math.round(capped * (0.5 + Math.random() * 0.5)) : capped;
680
680
  }
681
681
  __name(backoffDelay, "backoffDelay");
682
+ async function retryLocalStep(options, attempt) {
683
+ const maxAttempts = Math.max(1, options?.retries ?? 1);
684
+ for (let n = 1; ; n += 1) {
685
+ try {
686
+ return {
687
+ ok: true,
688
+ value: await attempt(n),
689
+ attempts: n
690
+ };
691
+ } catch (error2) {
692
+ if (error2 instanceof FatalError || n >= maxAttempts) return {
693
+ ok: false,
694
+ error: error2,
695
+ attempts: n
696
+ };
697
+ const wait = backoffDelay(n, options);
698
+ if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
699
+ }
700
+ }
701
+ }
702
+ __name(retryLocalStep, "retryLocalStep");
682
703
 
683
704
  // src/bounded-runner.ts
684
705
  var BoundedRunner = class {
@@ -3172,6 +3193,11 @@ var RemoteWorkflowExecutor = class {
3172
3193
  };
3173
3194
  await this.transport.dispatchWorkflowTask(task);
3174
3195
  }
3196
+ /** Where the broker has the task dispatched under `taskId`, when the transport can tell. */
3197
+ async taskState(_run, taskId) {
3198
+ if (!this.transport.workflowTaskState) return "unknown";
3199
+ return this.transport.workflowTaskState(tenantGroup(sanitizeQueueToken(this.name), this.partition), taskId);
3200
+ }
3175
3201
  };
3176
3202
 
3177
3203
  // src/run-dispatch-batcher.ts
@@ -5908,7 +5934,6 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
5908
5934
  });
5909
5935
  return existing.output;
5910
5936
  }
5911
- const maxAttempts = Math.max(1, options?.retries ?? 1);
5912
5937
  const startedAt = /* @__PURE__ */ new Date();
5913
5938
  await host.startStep({
5914
5939
  runId,
@@ -5919,60 +5944,59 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
5919
5944
  enqueuedAt: startedAt,
5920
5945
  startedAt
5921
5946
  });
5922
- for (let attempt = 1; ; attempt += 1) {
5923
- const events = [];
5924
- try {
5925
- const invocation = {
5926
- runId,
5927
- workflow,
5928
- stepName: name,
5929
- seq: current,
5930
- attempt
5931
- };
5932
- const body = /* @__PURE__ */ __name(() => {
5933
- const logger = createStepLogger(events, host.clock);
5934
- return runInStepScope(scopeOwner, name, () => runInStepLogger(logger, () => fn(logger)));
5935
- }, "body");
5936
- const output = host.interceptStep ? await host.interceptStep(invocation, body) : await body();
5937
- await host.completeStep({
5938
- runId,
5939
- seq: current,
5940
- name,
5941
- kind: "local",
5942
- output,
5943
- events,
5944
- attempts: attempt,
5945
- enqueuedAt: startedAt,
5946
- startedAt
5947
- });
5948
- if (options?.compensate) compensations.push({
5949
- name,
5950
- fn: options.compensate
5951
- });
5952
- return output;
5953
- } catch (err) {
5954
- if (err instanceof FatalError || attempt >= maxAttempts) {
5955
- await host.failStep({
5956
- runId,
5957
- seq: current,
5958
- name,
5959
- kind: "local",
5960
- error: {
5961
- message: err instanceof Error ? err.message : String(err),
5962
- stack: err instanceof Error ? err.stack : void 0,
5963
- ...codeAndDetails(err)
5964
- },
5965
- events,
5966
- attempts: attempt,
5967
- enqueuedAt: startedAt,
5968
- startedAt
5969
- });
5970
- throw err;
5971
- }
5972
- const wait = backoffDelay(attempt, options);
5973
- if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
5974
- }
5947
+ let events = [];
5948
+ const outcome = await retryLocalStep(options, async (attempt) => {
5949
+ const attemptEvents = [];
5950
+ events = attemptEvents;
5951
+ const invocation = {
5952
+ runId,
5953
+ workflow,
5954
+ stepName: name,
5955
+ seq: current,
5956
+ attempt
5957
+ };
5958
+ const body = /* @__PURE__ */ __name(() => {
5959
+ const logger = createStepLogger(attemptEvents, host.clock);
5960
+ return runInStepScope(scopeOwner, name, () => runInStepLogger(logger, () => fn(logger)));
5961
+ }, "body");
5962
+ const output = host.interceptStep ? await host.interceptStep(invocation, body) : await body();
5963
+ await host.completeStep({
5964
+ runId,
5965
+ seq: current,
5966
+ name,
5967
+ kind: "local",
5968
+ output,
5969
+ events: attemptEvents,
5970
+ attempts: attempt,
5971
+ enqueuedAt: startedAt,
5972
+ startedAt
5973
+ });
5974
+ return output;
5975
+ });
5976
+ if (outcome.ok) {
5977
+ if (options?.compensate) compensations.push({
5978
+ name,
5979
+ fn: options.compensate
5980
+ });
5981
+ return outcome.value;
5975
5982
  }
5983
+ const err = outcome.error;
5984
+ await host.failStep({
5985
+ runId,
5986
+ seq: current,
5987
+ name,
5988
+ kind: "local",
5989
+ error: {
5990
+ message: err instanceof Error ? err.message : String(err),
5991
+ stack: err instanceof Error ? err.stack : void 0,
5992
+ ...codeAndDetails(err)
5993
+ },
5994
+ events,
5995
+ attempts: outcome.attempts,
5996
+ enqueuedAt: startedAt,
5997
+ startedAt
5998
+ });
5999
+ throw err;
5976
6000
  }, "localStep");
5977
6001
  const transaction = /* @__PURE__ */ __name(async (name, fn) => {
5978
6002
  if (!store.transaction) {
@@ -6994,6 +7018,20 @@ function mergeTags(staticTags, runTags) {
6994
7018
  }
6995
7019
  __name(mergeTags, "mergeTags");
6996
7020
  var BREAKPOINT = "breakpoint";
7021
+ var LOST_TURN_SUFFIX = ":lost";
7022
+ var GRACED_TURN_SUFFIX = ":grace";
7023
+ function awaitsDecision(run, decision) {
7024
+ const marker = run.awaitingDecisionTaskId;
7025
+ return marker !== void 0 && awaitedTaskId(marker) === decision.taskId;
7026
+ }
7027
+ __name(awaitsDecision, "awaitsDecision");
7028
+ function awaitedTaskId(marker) {
7029
+ let id = marker;
7030
+ if (id.endsWith(LOST_TURN_SUFFIX)) id = id.slice(0, -LOST_TURN_SUFFIX.length);
7031
+ if (id.endsWith(GRACED_TURN_SUFFIX)) id = id.slice(0, -GRACED_TURN_SUFFIX.length);
7032
+ return id;
7033
+ }
7034
+ __name(awaitedTaskId, "awaitedTaskId");
6997
7035
  var PENDING_POLL_BATCH = 100;
6998
7036
  var DEFAULT_STICKY_RUNS = 1e3;
6999
7037
  var DEFAULT_STICKY_IDLE_MS = 3e5;
@@ -7263,6 +7301,9 @@ var WorkflowEngine = class _WorkflowEngine {
7263
7301
  * `clock() + leaseMs`, so at or after it). See {@link orphanedRuns}.
7264
7302
  */
7265
7303
  ownLeaseCutoffMs;
7304
+ /** When this engine was built (clock ms): {@link noteLapsedTurn} counts no lost turn within one
7305
+ * turn window of it, while live workers' beats catch up with deadlines that went stale. */
7306
+ bootedAtMs;
7266
7307
  maxRecoveryAttempts;
7267
7308
  unregisteredWorkflowGraceMs;
7268
7309
  remoteAdvanceSilenceMs;
@@ -7492,6 +7533,7 @@ var WorkflowEngine = class _WorkflowEngine {
7492
7533
  this.namespace = poolNamespace(deps.namespace);
7493
7534
  this.leaseMs = deps.leaseMs ?? 3e4;
7494
7535
  this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
7536
+ this.bootedAtMs = this.clock();
7495
7537
  this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
7496
7538
  const grace = deps.unregisteredWorkflowGraceMs;
7497
7539
  this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
@@ -7869,6 +7911,9 @@ var WorkflowEngine = class _WorkflowEngine {
7869
7911
  return;
7870
7912
  }
7871
7913
  if (dispatch.reason !== "wake" || TERMINAL_RUN_STATUSES.includes(run.status)) return;
7914
+ if (dispatch.awaiting !== void 0 && (run.awaitingDecisionTaskId === void 0 || awaitedTaskId(run.awaitingDecisionTaskId) !== dispatch.awaiting)) {
7915
+ return;
7916
+ }
7872
7917
  const nowMs = this.clock();
7873
7918
  if (run.wakeAt !== void 0 && run.wakeAt <= nowMs) {
7874
7919
  const waiters = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -7920,6 +7965,7 @@ var WorkflowEngine = class _WorkflowEngine {
7920
7965
  */
7921
7966
  async handOffDue(runs, nowMs, settled) {
7922
7967
  const claimed = [];
7968
+ const awaiting = /* @__PURE__ */ new Map();
7923
7969
  for (const run of runs) {
7924
7970
  if (this.draining) break;
7925
7971
  this.markClaim(run.id);
@@ -7934,16 +7980,26 @@ var WorkflowEngine = class _WorkflowEngine {
7934
7980
  continue;
7935
7981
  }
7936
7982
  try {
7983
+ const seen = {};
7984
+ const handled = await this.noteLapsedTurn(run.id, nowMs, seen);
7985
+ if (handled) {
7986
+ if (handled.status === "dead") settled?.set(run.id, handled);
7987
+ continue;
7988
+ }
7937
7989
  const patch = {
7938
7990
  wakeAt: nowMs + this.leaseMs
7939
7991
  };
7992
+ let won = true;
7940
7993
  if (this.store.updateRunIf) {
7941
- if (await this.store.updateRunIf(run.id, [
7994
+ won = await this.store.updateRunIf(run.id, [
7942
7995
  run.status
7943
- ], patch)) claimed.push(run);
7996
+ ], patch);
7944
7997
  } else {
7945
7998
  await this.store.updateRun(run.id, patch);
7999
+ }
8000
+ if (won) {
7946
8001
  claimed.push(run);
8002
+ if (seen.marker !== void 0) awaiting.set(run.id, awaitedTaskId(seen.marker));
7947
8003
  }
7948
8004
  } finally {
7949
8005
  await this.store.releaseRunLock(run.id, this.instanceId);
@@ -7954,13 +8010,19 @@ var WorkflowEngine = class _WorkflowEngine {
7954
8010
  }
7955
8011
  try {
7956
8012
  for (let at = 0; at < claimed.length; at += RUN_DISPATCH_BATCH) {
7957
- await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => ({
7958
- runId: run.id,
7959
- reason: "wake",
7960
- ...run.priority !== void 0 ? {
7961
- priority: run.priority
7962
- } : {}
7963
- })), {
8013
+ await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => {
8014
+ const turn = awaiting.get(run.id);
8015
+ return {
8016
+ runId: run.id,
8017
+ reason: "wake",
8018
+ ...run.priority !== void 0 ? {
8019
+ priority: run.priority
8020
+ } : {},
8021
+ ...turn !== void 0 ? {
8022
+ awaiting: turn
8023
+ } : {}
8024
+ };
8025
+ }), {
7964
8026
  namespace: this.namespace
7965
8027
  });
7966
8028
  }
@@ -11034,28 +11096,39 @@ var WorkflowEngine = class _WorkflowEngine {
11034
11096
  * `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
11035
11097
  * result to skip the resume, or `undefined` to proceed.
11036
11098
  */
11037
- async countRecovery(run) {
11099
+ async countRecovery(run, extra = {}) {
11038
11100
  const attempts = (run.recoveryAttempts ?? 0) + 1;
11101
+ const updateIf = /* @__PURE__ */ __name((patch) => (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11102
+ run.status
11103
+ ], patch), "updateIf");
11104
+ const moved = /* @__PURE__ */ __name(async () => {
11105
+ await this.store.releaseRunLock(run.id);
11106
+ return this.echoCurrentStatus(run.id);
11107
+ }, "moved");
11039
11108
  if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
11040
11109
  const error2 = {
11041
11110
  message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
11042
11111
  code: "max_recovery_attempts"
11043
11112
  };
11044
- await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
11113
+ const written = await updateIf(await this.withLifecycleMarker(run, {
11045
11114
  status: "dead",
11046
11115
  error: error2,
11047
11116
  updatedAt: /* @__PURE__ */ new Date()
11048
11117
  }));
11118
+ if (!written) return moved();
11049
11119
  await this.store.releaseRunLock(run.id);
11050
11120
  return this.announceDead({
11051
11121
  ...run,
11052
11122
  recoveryAttempts: attempts
11053
11123
  }, error2);
11054
11124
  }
11055
- await this.store.updateRun(run.id, {
11125
+ if (!await updateIf({
11126
+ ...extra,
11056
11127
  recoveryAttempts: attempts,
11057
11128
  updatedAt: /* @__PURE__ */ new Date()
11058
- });
11129
+ })) {
11130
+ return moved();
11131
+ }
11059
11132
  return void 0;
11060
11133
  }
11061
11134
  /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
@@ -11202,7 +11275,87 @@ var WorkflowEngine = class _WorkflowEngine {
11202
11275
  return this.resumeLeased([
11203
11276
  ...due,
11204
11277
  ...blocked
11205
- ], nowMs, (run) => this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11278
+ ], nowMs, async (run) => await this.noteLapsedTurn(run.id, nowMs) ?? this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11279
+ }
11280
+ /**
11281
+ * Called under the run's lease by the due-timer pass, for a run that THIS instance would re-drive.
11282
+ * A run suspended awaiting a DISPATCHED workflow turn's decision (`awaitingDecisionTaskId`) whose
11283
+ * re-drive deadline (`wakeAt`) has passed lost that turn: no decision came back and the worker
11284
+ * stopped beating (every beat pushes `wakeAt` forward, see {@link rearmDecisionDeadline}), so the
11285
+ * worker running it died, or the task was lost with it. That is the dispatched form of a crashed
11286
+ * inline turn, so it spends the same {@link maxRecoveryAttempts} budget through
11287
+ * {@link countRecovery}: a turn that keeps killing its worker ends `dead` (with the same
11288
+ * `run.failed` announcement) instead of being re-dispatched, and redelivered, for ever.
11289
+ *
11290
+ * Counted at most once per awaited turn: the count rewrites the marker to its `:lost` form, which
11291
+ * no later pass counts again however long the re-drive waits on the run queue, and the re-drive's
11292
+ * dispatch replaces it with the next turn's id. Not counted, and the re-drive left for later:
11293
+ * - while the broker still holds the task unstarted (a backed-up queue, not a dead worker), or
11294
+ * - within one turn window of this engine's boot, when beats from live workers have not yet had
11295
+ * the chance to push a deadline that went stale while the engine was down. Once per turn (the
11296
+ * `:grace` marker): a lapse after that takes the normal path, whichever instance judges it.
11297
+ * A late decision of a counted turn is still applied while the re-drive has not dispatched yet
11298
+ * (see `awaitsDecision`), and gives the attempt back.
11299
+ * Never counted by an instance that cannot resolve the workflow (it re-drives nothing), nor for a
11300
+ * decision that arrives, a worker that keeps beating, or any write a healthy turn makes on its way.
11301
+ *
11302
+ * Re-reads under the lease: the listing may predate a decision applied since. Returns a result when
11303
+ * the run was dead-lettered or left waiting (the lease released either way), else `undefined` for
11304
+ * the caller to re-drive as before. A decision that lands while this holds the lease is dropped (it
11305
+ * cannot take the lease), whichever way this pass goes: with a run it dead-letters, or with one it
11306
+ * graces or leaves waiting on a `queued` task, whose next lapse then counts the turn and runs it
11307
+ * again. A narrow race, accepted.
11308
+ *
11309
+ * The marker grows by up to 11 characters (`:grace:lost`) beyond the dispatched id, which is
11310
+ * `<runId>:wf:<8 hex>`: `runId` plus 23 in all. A store that bounds the column (TypeORM's
11311
+ * `varchar(191)`) therefore dispatches a run whose id is 169 to 179 characters long, then refuses
11312
+ * the grace or lost write for it.
11313
+ */
11314
+ async noteLapsedTurn(runId, nowMs, seen) {
11315
+ const run = await this.store.getRun(runId);
11316
+ const awaited = run?.awaitingDecisionTaskId;
11317
+ if (seen) seen.marker = awaited;
11318
+ if (!run || run.status !== "suspended" || awaited === void 0 || awaited.endsWith(LOST_TURN_SUFFIX) || run.wakeAt == null || run.wakeAt > nowMs) {
11319
+ return void 0;
11320
+ }
11321
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) return void 0;
11322
+ const windowMs = this.dispatchedTurnSilenceMs;
11323
+ if (windowMs != null && nowMs < this.bootedAtMs + windowMs && !awaited.endsWith(GRACED_TURN_SUFFIX)) {
11324
+ return this.leaveTurnWaiting(run, this.bootedAtMs + windowMs, {
11325
+ awaitingDecisionTaskId: `${awaited}${GRACED_TURN_SUFFIX}`
11326
+ });
11327
+ }
11328
+ const state = await this.dispatchedTurnState(run, awaitedTaskId(awaited));
11329
+ if (state === "queued") return this.leaveTurnWaiting(run, nowMs + (windowMs ?? this.leaseMs));
11330
+ if (state === "error") return void 0;
11331
+ return this.countRecovery(run, {
11332
+ awaitingDecisionTaskId: `${awaited}${LOST_TURN_SUFFIX}`
11333
+ });
11334
+ }
11335
+ /** Where the broker has the task of a dispatched turn: see {@link WorkflowExecutor.taskState}. */
11336
+ async dispatchedTurnState(run, taskId) {
11337
+ const executor = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.remote?.executor;
11338
+ if (!executor?.taskState) return "unknown";
11339
+ try {
11340
+ return await executor.taskState(run, taskId);
11341
+ } catch {
11342
+ return "error";
11343
+ }
11344
+ }
11345
+ /** Push a lapsed turn's deadline to `until` without re-driving it, and give the lease back. */
11346
+ async leaveTurnWaiting(run, until, extra = {}) {
11347
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11348
+ "suspended"
11349
+ ], {
11350
+ ...extra,
11351
+ wakeAt: until,
11352
+ updatedAt: /* @__PURE__ */ new Date()
11353
+ });
11354
+ await this.store.releaseRunLock(run.id);
11355
+ return {
11356
+ runId: run.id,
11357
+ status: "suspended"
11358
+ };
11206
11359
  }
11207
11360
  /**
11208
11361
  * The reconcile safety net's per-run check (see {@link resumeDueTimers}): if this suspended run has
@@ -13504,6 +13657,12 @@ ${run.workflowVersion}`;
13504
13657
  const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
13505
13658
  status: "suspended",
13506
13659
  wakeAt: this.reconcileWakeAt(outcome.wakeAt),
13660
+ // A turn that ran here suspended the run, so no dispatched turn is awaited any more: a marker
13661
+ // left by an earlier dispatch (a re-drive that followed an onError handoff, say) would make
13662
+ // this suspend read as a lost turn at its next reconcile wake.
13663
+ ...run.awaitingDecisionTaskId !== void 0 ? {
13664
+ awaitingDecisionTaskId: void 0
13665
+ } : {},
13507
13666
  updatedAt: stamp
13508
13667
  }));
13509
13668
  if (!applied) return this.echoCurrentStatus(run.id);
@@ -14101,7 +14260,7 @@ ${run.workflowVersion}`;
14101
14260
  output: cmd.output,
14102
14261
  error: cmd.error,
14103
14262
  events: cmd.events,
14104
- attempts: 1,
14263
+ attempts: cmd.attempts ?? 1,
14105
14264
  enqueuedAt: startedAt,
14106
14265
  startedAt,
14107
14266
  finishedAt,
@@ -16326,7 +16485,7 @@ ${run.workflowVersion}`;
16326
16485
  */
16327
16486
  async completeRemoteDecision(decision) {
16328
16487
  const run = await this.store.getRun(decision.runId);
16329
- if (!run || run.awaitingDecisionTaskId !== decision.taskId) return;
16488
+ if (!run || !awaitsDecision(run, decision)) return;
16330
16489
  if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
16331
16490
  if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
16332
16491
  return;
@@ -16338,16 +16497,20 @@ ${run.workflowVersion}`;
16338
16497
  const held = this.holdExecutionLease(decision.runId);
16339
16498
  try {
16340
16499
  const fresh = await this.store.getRun(decision.runId);
16341
- if (!fresh || fresh.awaitingDecisionTaskId !== decision.taskId) return;
16500
+ if (!fresh || !awaitsDecision(fresh, decision)) return;
16342
16501
  if (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead") {
16343
16502
  return;
16344
16503
  }
16345
16504
  await this.store.updateRun(decision.runId, {
16346
16505
  awaitingDecisionTaskId: void 0,
16347
16506
  wakeAt: void 0,
16507
+ ...fresh.recoveryAttempts ? {
16508
+ recoveryAttempts: 0
16509
+ } : {},
16348
16510
  updatedAt: /* @__PURE__ */ new Date()
16349
16511
  });
16350
16512
  fresh.awaitingDecisionTaskId = void 0;
16513
+ if (fresh.recoveryAttempts) fresh.recoveryAttempts = 0;
16351
16514
  await this.applyDecision(fresh, decision);
16352
16515
  } finally {
16353
16516
  const fenced = held.lease.lost;
@@ -16383,6 +16546,9 @@ ${run.workflowVersion}`;
16383
16546
  this.decisionDeadlineRearms.delete(runId);
16384
16547
  return;
16385
16548
  }
16549
+ if (run.awaitingDecisionTaskId.endsWith(GRACED_TURN_SUFFIX) && !await this.clearTurnGrace(runId, run.awaitingDecisionTaskId, now)) {
16550
+ return;
16551
+ }
16386
16552
  const halfWindow = Math.ceil(silenceMs / 2);
16387
16553
  if (run.wakeAt != null && run.wakeAt > now + halfWindow) {
16388
16554
  this.decisionDeadlineRearms.set(runId, run.wakeAt - halfWindow);
@@ -16399,6 +16565,29 @@ ${run.workflowVersion}`;
16399
16565
  }
16400
16566
  }
16401
16567
  /**
16568
+ * Write a graced turn's marker back to its bare task id (see {@link GRACED_TURN_SUFFIX}), under the
16569
+ * run's lease and only while the marker is still `marker`: a decision and a re-dispatch both write
16570
+ * the marker under that lease, so this never puts back the id of a turn they already replaced.
16571
+ * False when the lease was busy or the write failed; the caller's next beat tries again.
16572
+ */
16573
+ async clearTurnGrace(runId, marker, nowMs) {
16574
+ if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
16575
+ return false;
16576
+ }
16577
+ try {
16578
+ const fresh = await this.store.getRun(runId);
16579
+ if (fresh?.awaitingDecisionTaskId === marker) {
16580
+ await this.store.updateRun(runId, {
16581
+ awaitingDecisionTaskId: marker.slice(0, -GRACED_TURN_SUFFIX.length),
16582
+ updatedAt: /* @__PURE__ */ new Date()
16583
+ });
16584
+ }
16585
+ return true;
16586
+ } finally {
16587
+ await this.store.releaseRunLock(runId);
16588
+ }
16589
+ }
16590
+ /**
16402
16591
  * A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they
16403
16592
  * re-contend now instead of at their retry tick. Snapshot-and-clear — a run still blocked after the
16404
16593
  * retry re-registers itself, and one that's gone (cancelled/admitted) is dropped. Best-effort.
@@ -18794,6 +18983,7 @@ var InMemoryRunQueue = class {
18794
18983
  const waiting = lane.queued.get(run.runId);
18795
18984
  if (waiting) {
18796
18985
  if (run.reason === "wake") waiting.reason = "wake";
18986
+ if (waiting.awaiting !== run.awaiting) waiting.awaiting = void 0;
18797
18987
  continue;
18798
18988
  }
18799
18989
  const entry = {
@@ -19131,6 +19321,7 @@ export {
19131
19321
  resolveWireTenant,
19132
19322
  retentionKeeps,
19133
19323
  retiredHandoffName,
19324
+ retryLocalStep,
19134
19325
  routeFor,
19135
19326
  rowTenant,
19136
19327
  runCompletedEvent,