@dudousxd/nestjs-durable-core 0.104.0 → 0.106.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -246,6 +246,7 @@ __export(index_exports, {
246
246
  resolveWireTenant: () => resolveWireTenant,
247
247
  retentionKeeps: () => retentionKeeps,
248
248
  retiredHandoffName: () => retiredHandoffName,
249
+ retryLocalStep: () => retryLocalStep,
249
250
  routeFor: () => routeFor,
250
251
  rowTenant: () => rowTenant,
251
252
  runCompletedEvent: () => runCompletedEvent,
@@ -981,6 +982,27 @@ function backoffDelay(attempt, options) {
981
982
  return options?.jitter ? Math.round(capped * (0.5 + Math.random() * 0.5)) : capped;
982
983
  }
983
984
  __name(backoffDelay, "backoffDelay");
985
+ async function retryLocalStep(options, attempt) {
986
+ const maxAttempts = Math.max(1, options?.retries ?? 1);
987
+ for (let n = 1; ; n += 1) {
988
+ try {
989
+ return {
990
+ ok: true,
991
+ value: await attempt(n),
992
+ attempts: n
993
+ };
994
+ } catch (error2) {
995
+ if (error2 instanceof FatalError || n >= maxAttempts) return {
996
+ ok: false,
997
+ error: error2,
998
+ attempts: n
999
+ };
1000
+ const wait = backoffDelay(n, options);
1001
+ if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
1002
+ }
1003
+ }
1004
+ }
1005
+ __name(retryLocalStep, "retryLocalStep");
984
1006
 
985
1007
  // src/bounded-runner.ts
986
1008
  var BoundedRunner = class {
@@ -3474,6 +3496,11 @@ var RemoteWorkflowExecutor = class {
3474
3496
  };
3475
3497
  await this.transport.dispatchWorkflowTask(task);
3476
3498
  }
3499
+ /** Where the broker has the task dispatched under `taskId`, when the transport can tell. */
3500
+ async taskState(_run, taskId) {
3501
+ if (!this.transport.workflowTaskState) return "unknown";
3502
+ return this.transport.workflowTaskState(tenantGroup(sanitizeQueueToken(this.name), this.partition), taskId);
3503
+ }
3477
3504
  };
3478
3505
 
3479
3506
  // src/run-dispatch-batcher.ts
@@ -6210,7 +6237,6 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
6210
6237
  });
6211
6238
  return existing.output;
6212
6239
  }
6213
- const maxAttempts = Math.max(1, options?.retries ?? 1);
6214
6240
  const startedAt = /* @__PURE__ */ new Date();
6215
6241
  await host.startStep({
6216
6242
  runId,
@@ -6221,60 +6247,59 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
6221
6247
  enqueuedAt: startedAt,
6222
6248
  startedAt
6223
6249
  });
6224
- for (let attempt = 1; ; attempt += 1) {
6225
- const events = [];
6226
- try {
6227
- const invocation = {
6228
- runId,
6229
- workflow,
6230
- stepName: name,
6231
- seq: current,
6232
- attempt
6233
- };
6234
- const body = /* @__PURE__ */ __name(() => {
6235
- const logger = createStepLogger(events, host.clock);
6236
- return runInStepScope(scopeOwner, name, () => runInStepLogger(logger, () => fn(logger)));
6237
- }, "body");
6238
- const output = host.interceptStep ? await host.interceptStep(invocation, body) : await body();
6239
- await host.completeStep({
6240
- runId,
6241
- seq: current,
6242
- name,
6243
- kind: "local",
6244
- output,
6245
- events,
6246
- attempts: attempt,
6247
- enqueuedAt: startedAt,
6248
- startedAt
6249
- });
6250
- if (options?.compensate) compensations.push({
6251
- name,
6252
- fn: options.compensate
6253
- });
6254
- return output;
6255
- } catch (err) {
6256
- if (err instanceof FatalError || attempt >= maxAttempts) {
6257
- await host.failStep({
6258
- runId,
6259
- seq: current,
6260
- name,
6261
- kind: "local",
6262
- error: {
6263
- message: err instanceof Error ? err.message : String(err),
6264
- stack: err instanceof Error ? err.stack : void 0,
6265
- ...codeAndDetails(err)
6266
- },
6267
- events,
6268
- attempts: attempt,
6269
- enqueuedAt: startedAt,
6270
- startedAt
6271
- });
6272
- throw err;
6273
- }
6274
- const wait = backoffDelay(attempt, options);
6275
- if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
6276
- }
6250
+ let events = [];
6251
+ const outcome = await retryLocalStep(options, async (attempt) => {
6252
+ const attemptEvents = [];
6253
+ events = attemptEvents;
6254
+ const invocation = {
6255
+ runId,
6256
+ workflow,
6257
+ stepName: name,
6258
+ seq: current,
6259
+ attempt
6260
+ };
6261
+ const body = /* @__PURE__ */ __name(() => {
6262
+ const logger = createStepLogger(attemptEvents, host.clock);
6263
+ return runInStepScope(scopeOwner, name, () => runInStepLogger(logger, () => fn(logger)));
6264
+ }, "body");
6265
+ const output = host.interceptStep ? await host.interceptStep(invocation, body) : await body();
6266
+ await host.completeStep({
6267
+ runId,
6268
+ seq: current,
6269
+ name,
6270
+ kind: "local",
6271
+ output,
6272
+ events: attemptEvents,
6273
+ attempts: attempt,
6274
+ enqueuedAt: startedAt,
6275
+ startedAt
6276
+ });
6277
+ return output;
6278
+ });
6279
+ if (outcome.ok) {
6280
+ if (options?.compensate) compensations.push({
6281
+ name,
6282
+ fn: options.compensate
6283
+ });
6284
+ return outcome.value;
6277
6285
  }
6286
+ const err = outcome.error;
6287
+ await host.failStep({
6288
+ runId,
6289
+ seq: current,
6290
+ name,
6291
+ kind: "local",
6292
+ error: {
6293
+ message: err instanceof Error ? err.message : String(err),
6294
+ stack: err instanceof Error ? err.stack : void 0,
6295
+ ...codeAndDetails(err)
6296
+ },
6297
+ events,
6298
+ attempts: outcome.attempts,
6299
+ enqueuedAt: startedAt,
6300
+ startedAt
6301
+ });
6302
+ throw err;
6278
6303
  }, "localStep");
6279
6304
  const transaction = /* @__PURE__ */ __name(async (name, fn) => {
6280
6305
  if (!store.transaction) {
@@ -7296,6 +7321,20 @@ function mergeTags(staticTags, runTags) {
7296
7321
  }
7297
7322
  __name(mergeTags, "mergeTags");
7298
7323
  var BREAKPOINT = "breakpoint";
7324
+ var LOST_TURN_SUFFIX = ":lost";
7325
+ var GRACED_TURN_SUFFIX = ":grace";
7326
+ function awaitsDecision(run, decision) {
7327
+ const marker = run.awaitingDecisionTaskId;
7328
+ return marker !== void 0 && awaitedTaskId(marker) === decision.taskId;
7329
+ }
7330
+ __name(awaitsDecision, "awaitsDecision");
7331
+ function awaitedTaskId(marker) {
7332
+ let id = marker;
7333
+ if (id.endsWith(LOST_TURN_SUFFIX)) id = id.slice(0, -LOST_TURN_SUFFIX.length);
7334
+ if (id.endsWith(GRACED_TURN_SUFFIX)) id = id.slice(0, -GRACED_TURN_SUFFIX.length);
7335
+ return id;
7336
+ }
7337
+ __name(awaitedTaskId, "awaitedTaskId");
7299
7338
  var PENDING_POLL_BATCH = 100;
7300
7339
  var DEFAULT_STICKY_RUNS = 1e3;
7301
7340
  var DEFAULT_STICKY_IDLE_MS = 3e5;
@@ -7565,6 +7604,9 @@ var WorkflowEngine = class _WorkflowEngine {
7565
7604
  * `clock() + leaseMs`, so at or after it). See {@link orphanedRuns}.
7566
7605
  */
7567
7606
  ownLeaseCutoffMs;
7607
+ /** When this engine was built (clock ms): {@link noteLapsedTurn} counts no lost turn within one
7608
+ * turn window of it, while live workers' beats catch up with deadlines that went stale. */
7609
+ bootedAtMs;
7568
7610
  maxRecoveryAttempts;
7569
7611
  unregisteredWorkflowGraceMs;
7570
7612
  remoteAdvanceSilenceMs;
@@ -7794,6 +7836,7 @@ var WorkflowEngine = class _WorkflowEngine {
7794
7836
  this.namespace = poolNamespace(deps.namespace);
7795
7837
  this.leaseMs = deps.leaseMs ?? 3e4;
7796
7838
  this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
7839
+ this.bootedAtMs = this.clock();
7797
7840
  this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
7798
7841
  const grace = deps.unregisteredWorkflowGraceMs;
7799
7842
  this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
@@ -8171,6 +8214,9 @@ var WorkflowEngine = class _WorkflowEngine {
8171
8214
  return;
8172
8215
  }
8173
8216
  if (dispatch.reason !== "wake" || TERMINAL_RUN_STATUSES.includes(run.status)) return;
8217
+ if (dispatch.awaiting !== void 0 && (run.awaitingDecisionTaskId === void 0 || awaitedTaskId(run.awaitingDecisionTaskId) !== dispatch.awaiting)) {
8218
+ return;
8219
+ }
8174
8220
  const nowMs = this.clock();
8175
8221
  if (run.wakeAt !== void 0 && run.wakeAt <= nowMs) {
8176
8222
  const waiters = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -8222,6 +8268,7 @@ var WorkflowEngine = class _WorkflowEngine {
8222
8268
  */
8223
8269
  async handOffDue(runs, nowMs, settled) {
8224
8270
  const claimed = [];
8271
+ const awaiting = /* @__PURE__ */ new Map();
8225
8272
  for (const run of runs) {
8226
8273
  if (this.draining) break;
8227
8274
  this.markClaim(run.id);
@@ -8236,16 +8283,26 @@ var WorkflowEngine = class _WorkflowEngine {
8236
8283
  continue;
8237
8284
  }
8238
8285
  try {
8286
+ const seen = {};
8287
+ const handled = await this.noteLapsedTurn(run.id, nowMs, seen);
8288
+ if (handled) {
8289
+ if (handled.status === "dead") settled?.set(run.id, handled);
8290
+ continue;
8291
+ }
8239
8292
  const patch = {
8240
8293
  wakeAt: nowMs + this.leaseMs
8241
8294
  };
8295
+ let won = true;
8242
8296
  if (this.store.updateRunIf) {
8243
- if (await this.store.updateRunIf(run.id, [
8297
+ won = await this.store.updateRunIf(run.id, [
8244
8298
  run.status
8245
- ], patch)) claimed.push(run);
8299
+ ], patch);
8246
8300
  } else {
8247
8301
  await this.store.updateRun(run.id, patch);
8302
+ }
8303
+ if (won) {
8248
8304
  claimed.push(run);
8305
+ if (seen.marker !== void 0) awaiting.set(run.id, awaitedTaskId(seen.marker));
8249
8306
  }
8250
8307
  } finally {
8251
8308
  await this.store.releaseRunLock(run.id, this.instanceId);
@@ -8256,13 +8313,19 @@ var WorkflowEngine = class _WorkflowEngine {
8256
8313
  }
8257
8314
  try {
8258
8315
  for (let at = 0; at < claimed.length; at += RUN_DISPATCH_BATCH) {
8259
- await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => ({
8260
- runId: run.id,
8261
- reason: "wake",
8262
- ...run.priority !== void 0 ? {
8263
- priority: run.priority
8264
- } : {}
8265
- })), {
8316
+ await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => {
8317
+ const turn = awaiting.get(run.id);
8318
+ return {
8319
+ runId: run.id,
8320
+ reason: "wake",
8321
+ ...run.priority !== void 0 ? {
8322
+ priority: run.priority
8323
+ } : {},
8324
+ ...turn !== void 0 ? {
8325
+ awaiting: turn
8326
+ } : {}
8327
+ };
8328
+ }), {
8266
8329
  namespace: this.namespace
8267
8330
  });
8268
8331
  }
@@ -11336,28 +11399,39 @@ var WorkflowEngine = class _WorkflowEngine {
11336
11399
  * `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
11337
11400
  * result to skip the resume, or `undefined` to proceed.
11338
11401
  */
11339
- async countRecovery(run) {
11402
+ async countRecovery(run, extra = {}) {
11340
11403
  const attempts = (run.recoveryAttempts ?? 0) + 1;
11404
+ const updateIf = /* @__PURE__ */ __name((patch) => (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11405
+ run.status
11406
+ ], patch), "updateIf");
11407
+ const moved = /* @__PURE__ */ __name(async () => {
11408
+ await this.store.releaseRunLock(run.id);
11409
+ return this.echoCurrentStatus(run.id);
11410
+ }, "moved");
11341
11411
  if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
11342
11412
  const error2 = {
11343
11413
  message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
11344
11414
  code: "max_recovery_attempts"
11345
11415
  };
11346
- await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
11416
+ const written = await updateIf(await this.withLifecycleMarker(run, {
11347
11417
  status: "dead",
11348
11418
  error: error2,
11349
11419
  updatedAt: /* @__PURE__ */ new Date()
11350
11420
  }));
11421
+ if (!written) return moved();
11351
11422
  await this.store.releaseRunLock(run.id);
11352
11423
  return this.announceDead({
11353
11424
  ...run,
11354
11425
  recoveryAttempts: attempts
11355
11426
  }, error2);
11356
11427
  }
11357
- await this.store.updateRun(run.id, {
11428
+ if (!await updateIf({
11429
+ ...extra,
11358
11430
  recoveryAttempts: attempts,
11359
11431
  updatedAt: /* @__PURE__ */ new Date()
11360
- });
11432
+ })) {
11433
+ return moved();
11434
+ }
11361
11435
  return void 0;
11362
11436
  }
11363
11437
  /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
@@ -11504,7 +11578,87 @@ var WorkflowEngine = class _WorkflowEngine {
11504
11578
  return this.resumeLeased([
11505
11579
  ...due,
11506
11580
  ...blocked
11507
- ], nowMs, (run) => this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11581
+ ], nowMs, async (run) => await this.noteLapsedTurn(run.id, nowMs) ?? this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
11582
+ }
11583
+ /**
11584
+ * Called under the run's lease by the due-timer pass, for a run that THIS instance would re-drive.
11585
+ * A run suspended awaiting a DISPATCHED workflow turn's decision (`awaitingDecisionTaskId`) whose
11586
+ * re-drive deadline (`wakeAt`) has passed lost that turn: no decision came back and the worker
11587
+ * stopped beating (every beat pushes `wakeAt` forward, see {@link rearmDecisionDeadline}), so the
11588
+ * worker running it died, or the task was lost with it. That is the dispatched form of a crashed
11589
+ * inline turn, so it spends the same {@link maxRecoveryAttempts} budget through
11590
+ * {@link countRecovery}: a turn that keeps killing its worker ends `dead` (with the same
11591
+ * `run.failed` announcement) instead of being re-dispatched, and redelivered, for ever.
11592
+ *
11593
+ * Counted at most once per awaited turn: the count rewrites the marker to its `:lost` form, which
11594
+ * no later pass counts again however long the re-drive waits on the run queue, and the re-drive's
11595
+ * dispatch replaces it with the next turn's id. Not counted, and the re-drive left for later:
11596
+ * - while the broker still holds the task unstarted (a backed-up queue, not a dead worker), or
11597
+ * - within one turn window of this engine's boot, when beats from live workers have not yet had
11598
+ * the chance to push a deadline that went stale while the engine was down. Once per turn (the
11599
+ * `:grace` marker): a lapse after that takes the normal path, whichever instance judges it.
11600
+ * A late decision of a counted turn is still applied while the re-drive has not dispatched yet
11601
+ * (see `awaitsDecision`), and gives the attempt back.
11602
+ * Never counted by an instance that cannot resolve the workflow (it re-drives nothing), nor for a
11603
+ * decision that arrives, a worker that keeps beating, or any write a healthy turn makes on its way.
11604
+ *
11605
+ * Re-reads under the lease: the listing may predate a decision applied since. Returns a result when
11606
+ * the run was dead-lettered or left waiting (the lease released either way), else `undefined` for
11607
+ * the caller to re-drive as before. A decision that lands while this holds the lease is dropped (it
11608
+ * cannot take the lease), whichever way this pass goes: with a run it dead-letters, or with one it
11609
+ * graces or leaves waiting on a `queued` task, whose next lapse then counts the turn and runs it
11610
+ * again. A narrow race, accepted.
11611
+ *
11612
+ * The marker grows by up to 11 characters (`:grace:lost`) beyond the dispatched id, which is
11613
+ * `<runId>:wf:<8 hex>`: `runId` plus 23 in all. A store that bounds the column (TypeORM's
11614
+ * `varchar(191)`) therefore dispatches a run whose id is 169 to 179 characters long, then refuses
11615
+ * the grace or lost write for it.
11616
+ */
11617
+ async noteLapsedTurn(runId, nowMs, seen) {
11618
+ const run = await this.store.getRun(runId);
11619
+ const awaited = run?.awaitingDecisionTaskId;
11620
+ if (seen) seen.marker = awaited;
11621
+ if (!run || run.status !== "suspended" || awaited === void 0 || awaited.endsWith(LOST_TURN_SUFFIX) || run.wakeAt == null || run.wakeAt > nowMs) {
11622
+ return void 0;
11623
+ }
11624
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) return void 0;
11625
+ const windowMs = this.dispatchedTurnSilenceMs;
11626
+ if (windowMs != null && nowMs < this.bootedAtMs + windowMs && !awaited.endsWith(GRACED_TURN_SUFFIX)) {
11627
+ return this.leaveTurnWaiting(run, this.bootedAtMs + windowMs, {
11628
+ awaitingDecisionTaskId: `${awaited}${GRACED_TURN_SUFFIX}`
11629
+ });
11630
+ }
11631
+ const state = await this.dispatchedTurnState(run, awaitedTaskId(awaited));
11632
+ if (state === "queued") return this.leaveTurnWaiting(run, nowMs + (windowMs ?? this.leaseMs));
11633
+ if (state === "error") return void 0;
11634
+ return this.countRecovery(run, {
11635
+ awaitingDecisionTaskId: `${awaited}${LOST_TURN_SUFFIX}`
11636
+ });
11637
+ }
11638
+ /** Where the broker has the task of a dispatched turn: see {@link WorkflowExecutor.taskState}. */
11639
+ async dispatchedTurnState(run, taskId) {
11640
+ const executor = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.remote?.executor;
11641
+ if (!executor?.taskState) return "unknown";
11642
+ try {
11643
+ return await executor.taskState(run, taskId);
11644
+ } catch {
11645
+ return "error";
11646
+ }
11647
+ }
11648
+ /** Push a lapsed turn's deadline to `until` without re-driving it, and give the lease back. */
11649
+ async leaveTurnWaiting(run, until, extra = {}) {
11650
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
11651
+ "suspended"
11652
+ ], {
11653
+ ...extra,
11654
+ wakeAt: until,
11655
+ updatedAt: /* @__PURE__ */ new Date()
11656
+ });
11657
+ await this.store.releaseRunLock(run.id);
11658
+ return {
11659
+ runId: run.id,
11660
+ status: "suspended"
11661
+ };
11508
11662
  }
11509
11663
  /**
11510
11664
  * The reconcile safety net's per-run check (see {@link resumeDueTimers}): if this suspended run has
@@ -13806,6 +13960,12 @@ ${run.workflowVersion}`;
13806
13960
  const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
13807
13961
  status: "suspended",
13808
13962
  wakeAt: this.reconcileWakeAt(outcome.wakeAt),
13963
+ // A turn that ran here suspended the run, so no dispatched turn is awaited any more: a marker
13964
+ // left by an earlier dispatch (a re-drive that followed an onError handoff, say) would make
13965
+ // this suspend read as a lost turn at its next reconcile wake.
13966
+ ...run.awaitingDecisionTaskId !== void 0 ? {
13967
+ awaitingDecisionTaskId: void 0
13968
+ } : {},
13809
13969
  updatedAt: stamp
13810
13970
  }));
13811
13971
  if (!applied) return this.echoCurrentStatus(run.id);
@@ -14403,7 +14563,7 @@ ${run.workflowVersion}`;
14403
14563
  output: cmd.output,
14404
14564
  error: cmd.error,
14405
14565
  events: cmd.events,
14406
- attempts: 1,
14566
+ attempts: cmd.attempts ?? 1,
14407
14567
  enqueuedAt: startedAt,
14408
14568
  startedAt,
14409
14569
  finishedAt,
@@ -16628,7 +16788,7 @@ ${run.workflowVersion}`;
16628
16788
  */
16629
16789
  async completeRemoteDecision(decision) {
16630
16790
  const run = await this.store.getRun(decision.runId);
16631
- if (!run || run.awaitingDecisionTaskId !== decision.taskId) return;
16791
+ if (!run || !awaitsDecision(run, decision)) return;
16632
16792
  if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
16633
16793
  if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
16634
16794
  return;
@@ -16640,16 +16800,20 @@ ${run.workflowVersion}`;
16640
16800
  const held = this.holdExecutionLease(decision.runId);
16641
16801
  try {
16642
16802
  const fresh = await this.store.getRun(decision.runId);
16643
- if (!fresh || fresh.awaitingDecisionTaskId !== decision.taskId) return;
16803
+ if (!fresh || !awaitsDecision(fresh, decision)) return;
16644
16804
  if (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead") {
16645
16805
  return;
16646
16806
  }
16647
16807
  await this.store.updateRun(decision.runId, {
16648
16808
  awaitingDecisionTaskId: void 0,
16649
16809
  wakeAt: void 0,
16810
+ ...fresh.recoveryAttempts ? {
16811
+ recoveryAttempts: 0
16812
+ } : {},
16650
16813
  updatedAt: /* @__PURE__ */ new Date()
16651
16814
  });
16652
16815
  fresh.awaitingDecisionTaskId = void 0;
16816
+ if (fresh.recoveryAttempts) fresh.recoveryAttempts = 0;
16653
16817
  await this.applyDecision(fresh, decision);
16654
16818
  } finally {
16655
16819
  const fenced = held.lease.lost;
@@ -16685,6 +16849,9 @@ ${run.workflowVersion}`;
16685
16849
  this.decisionDeadlineRearms.delete(runId);
16686
16850
  return;
16687
16851
  }
16852
+ if (run.awaitingDecisionTaskId.endsWith(GRACED_TURN_SUFFIX) && !await this.clearTurnGrace(runId, run.awaitingDecisionTaskId, now)) {
16853
+ return;
16854
+ }
16688
16855
  const halfWindow = Math.ceil(silenceMs / 2);
16689
16856
  if (run.wakeAt != null && run.wakeAt > now + halfWindow) {
16690
16857
  this.decisionDeadlineRearms.set(runId, run.wakeAt - halfWindow);
@@ -16701,6 +16868,29 @@ ${run.workflowVersion}`;
16701
16868
  }
16702
16869
  }
16703
16870
  /**
16871
+ * Write a graced turn's marker back to its bare task id (see {@link GRACED_TURN_SUFFIX}), under the
16872
+ * run's lease and only while the marker is still `marker`: a decision and a re-dispatch both write
16873
+ * the marker under that lease, so this never puts back the id of a turn they already replaced.
16874
+ * False when the lease was busy or the write failed; the caller's next beat tries again.
16875
+ */
16876
+ async clearTurnGrace(runId, marker, nowMs) {
16877
+ if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
16878
+ return false;
16879
+ }
16880
+ try {
16881
+ const fresh = await this.store.getRun(runId);
16882
+ if (fresh?.awaitingDecisionTaskId === marker) {
16883
+ await this.store.updateRun(runId, {
16884
+ awaitingDecisionTaskId: marker.slice(0, -GRACED_TURN_SUFFIX.length),
16885
+ updatedAt: /* @__PURE__ */ new Date()
16886
+ });
16887
+ }
16888
+ return true;
16889
+ } finally {
16890
+ await this.store.releaseRunLock(runId);
16891
+ }
16892
+ }
16893
+ /**
16704
16894
  * A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they
16705
16895
  * re-contend now instead of at their retry tick. Snapshot-and-clear — a run still blocked after the
16706
16896
  * retry re-registers itself, and one that's gone (cancelled/admitted) is dropped. Best-effort.
@@ -19096,6 +19286,7 @@ var InMemoryRunQueue = class {
19096
19286
  const waiting = lane.queued.get(run.runId);
19097
19287
  if (waiting) {
19098
19288
  if (run.reason === "wake") waiting.reason = "wake";
19289
+ if (waiting.awaiting !== run.awaiting) waiting.awaiting = void 0;
19099
19290
  continue;
19100
19291
  }
19101
19292
  const entry = {
@@ -19434,6 +19625,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
19434
19625
  resolveWireTenant,
19435
19626
  retentionKeeps,
19436
19627
  retiredHandoffName,
19628
+ retryLocalStep,
19437
19629
  routeFor,
19438
19630
  rowTenant,
19439
19631
  runCompletedEvent,