@dudousxd/nestjs-durable-core 0.102.0 → 0.103.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -881,6 +881,14 @@ function isNonEmptyRunScope(scope) {
881
881
  return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
882
882
  }
883
883
  __name(isNonEmptyRunScope, "isNonEmptyRunScope");
884
+ var REMOTE_WORKFLOW_FEATURES = [
885
+ "callPolicy",
886
+ "sleepUntil",
887
+ "waitSignalTimeout",
888
+ "waitEvent",
889
+ "spawnChild",
890
+ "continueAsNew"
891
+ ];
884
892
  var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
885
893
 
886
894
  // src/events.ts
@@ -3069,6 +3077,35 @@ function codeAndDetails(err) {
3069
3077
  }
3070
3078
  __name(codeAndDetails, "codeAndDetails");
3071
3079
 
3080
+ // src/remote-call-policy.ts
3081
+ function remoteCallPolicyOf(config, opts) {
3082
+ const policy = {};
3083
+ const retries = opts?.retries ?? config?.retries;
3084
+ if (retries !== void 0) policy.retries = retries;
3085
+ const backoff = opts?.backoff ?? config?.backoff;
3086
+ if (backoff !== void 0) policy.backoff = backoff;
3087
+ const backoffMs = opts?.backoffMs ?? config?.backoffMs;
3088
+ if (backoffMs !== void 0) policy.backoffMs = backoffMs;
3089
+ const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
3090
+ if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
3091
+ const jitter = opts?.jitter ?? config?.jitter;
3092
+ if (jitter !== void 0) policy.jitter = jitter;
3093
+ const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
3094
+ if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
3095
+ return Object.keys(policy).length > 0 ? policy : void 0;
3096
+ }
3097
+ __name(remoteCallPolicyOf, "remoteCallPolicyOf");
3098
+ function remoteCallMaxAttempts(policy) {
3099
+ return Math.max(1, policy?.retries ?? 1);
3100
+ }
3101
+ __name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
3102
+ function remoteCallRetries(policy, attempts, error2) {
3103
+ if (policy === void 0 || policy.timeoutMs) return false;
3104
+ if (error2?.retryable === false) return false;
3105
+ return attempts < remoteCallMaxAttempts(policy);
3106
+ }
3107
+ __name(remoteCallRetries, "remoteCallRetries");
3108
+
3072
3109
  // src/remote-event.ts
3073
3110
  var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
3074
3111
  function markRemoteEvent(event) {
@@ -3126,7 +3163,12 @@ var RemoteWorkflowExecutor = class {
3126
3163
  } : {},
3127
3164
  group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
3128
3165
  priority: run.priority,
3129
- attempt: 1
3166
+ attempt: 1,
3167
+ // What this control plane applies beyond the original command kinds, so a worker fails a turn
3168
+ // that needs more instead of emitting a command this engine would not understand.
3169
+ features: [
3170
+ ...REMOTE_WORKFLOW_FEATURES
3171
+ ]
3130
3172
  };
3131
3173
  await this.transport.dispatchWorkflowTask(task);
3132
3174
  }
@@ -13453,7 +13495,7 @@ ${run.workflowVersion}`;
13453
13495
  let wfPlan = await this.planRoute(wfToken, registered.requires);
13454
13496
  if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
13455
13497
  if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
13456
- const checkpoints = await this.replayCheckpoints(run.id);
13498
+ const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
13457
13499
  await this.deliverFinishedChildren(run.id, checkpoints);
13458
13500
  await this.deliverFinishedChildWaiters(run.id, checkpoints);
13459
13501
  const handoff = liveHandoff(run.id, checkpoints);
@@ -13539,6 +13581,10 @@ ${run.workflowVersion}`;
13539
13581
  */
13540
13582
  async applyDecision(run, decision) {
13541
13583
  if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
13584
+ const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
13585
+ if (handoff !== void 0 && decision.status === "completed") {
13586
+ return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
13587
+ }
13542
13588
  if (decision.status === "completed") {
13543
13589
  await this.applyCommands(run, decision.commands);
13544
13590
  return this.settleRun(run, {
@@ -13585,6 +13631,63 @@ ${run.workflowVersion}`;
13585
13631
  return suspended;
13586
13632
  }
13587
13633
  /**
13634
+ * A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
13635
+ * run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
13636
+ * engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
13637
+ * turn replayed stays cancelled and starts nothing.
13638
+ */
13639
+ async applyRemoteContinueAsNew(run, commands, input) {
13640
+ await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
13641
+ const fresh = await this.store.getRun(run.id);
13642
+ if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
13643
+ return {
13644
+ runId: run.id,
13645
+ status: fresh.status,
13646
+ output: fresh.output,
13647
+ error: fresh.error
13648
+ };
13649
+ }
13650
+ const nextId = nextContinuationId(run.id);
13651
+ try {
13652
+ await this.start(run.workflow, input, nextId, {
13653
+ namespace: run.namespace,
13654
+ ...inheritTenant(run)
13655
+ });
13656
+ } catch (err) {
13657
+ const message = err instanceof Error ? err.message : String(err);
13658
+ return this.settleRun(run, {
13659
+ kind: "failed",
13660
+ error: {
13661
+ message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
13662
+ code: "continue_as_new_failed"
13663
+ }
13664
+ });
13665
+ }
13666
+ await this.store.updateRun(run.id, {
13667
+ status: "completed",
13668
+ output: void 0,
13669
+ error: void 0,
13670
+ wakeAt: void 0,
13671
+ updatedAt: /* @__PURE__ */ new Date()
13672
+ });
13673
+ this.emit({
13674
+ type: "run.completed",
13675
+ runId: run.id,
13676
+ workflow: run.workflow,
13677
+ namespace: run.namespace
13678
+ }, {
13679
+ lifecycle: false
13680
+ });
13681
+ void this.notifyParent(run.id, {
13682
+ ok: true,
13683
+ value: void 0
13684
+ });
13685
+ return {
13686
+ runId: run.id,
13687
+ status: "completed"
13688
+ };
13689
+ }
13690
+ /**
13588
13691
  * Re-drive a run whose turn was computed from a history OLDER than the store's.
13589
13692
  *
13590
13693
  * A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
@@ -13602,11 +13705,12 @@ ${run.workflowVersion}`;
13602
13705
  * Scheduled off the current stack so it runs after the caller released the run lease.
13603
13706
  */
13604
13707
  async redriveIfTurnSawStaleHistory(runId, commands) {
13605
- const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
13708
+ const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
13606
13709
  if (blocking.length === 0) return;
13607
13710
  for (const cmd of blocking) {
13608
13711
  const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
13609
- if (checkpoint && checkpoint.status !== "pending") {
13712
+ const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
13713
+ if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
13610
13714
  setTimeout(() => {
13611
13715
  void this.resumeImplicit(runId).catch(() => void 0);
13612
13716
  }, 0).unref?.();
@@ -13855,7 +13959,12 @@ ${run.workflowVersion}`;
13855
13959
  // name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
13856
13960
  name: cp.kind === "signal" ? void 0 : cp.name,
13857
13961
  output: cp.status === "completed" ? cp.output : void 0,
13858
- error: cp.status === "failed" ? cp.error : void 0
13962
+ error: cp.status === "failed" ? cp.error : void 0,
13963
+ // How many times a dispatched step ran: a worker applying a `call` policy decides a retry
13964
+ // from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
13965
+ ...cp.kind === "remote" ? {
13966
+ attempts: cp.attempts
13967
+ } : {}
13859
13968
  });
13860
13969
  } else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
13861
13970
  events.push({
@@ -13912,11 +14021,12 @@ ${run.workflowVersion}`;
13912
14021
  } else if (cmd.kind === "call") {
13913
14022
  const existing = await this.store.getCheckpoint(run.id, cmd.seq);
13914
14023
  if (existing) {
13915
- const leaseAt = redriveLost ? await this.redriveLostCall(run, existing) : void 0;
14024
+ const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
14025
+ const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
13916
14026
  if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
13917
14027
  continue;
13918
14028
  }
13919
- const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14029
+ const lease = this.callLease(cmd.policy);
13920
14030
  await this.store.saveCheckpoint(stepCheckpoint({
13921
14031
  runId: run.id,
13922
14032
  seq: cmd.seq,
@@ -13976,35 +14086,7 @@ ${run.workflowVersion}`;
13976
14086
  }));
13977
14087
  wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
13978
14088
  } else if (cmd.kind === "waitSignal") {
13979
- const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
13980
- await this.store.saveCheckpoint(instantCheckpoint({
13981
- runId: run.id,
13982
- seq: cmd.seq,
13983
- name: `signal:${cmd.signal}`,
13984
- kind: "signal",
13985
- output: payload
13986
- }));
13987
- setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
13988
- }, "deliverBuffered");
13989
- const buffered = await this.store.takeBufferedSignal(cmd.signal);
13990
- if (buffered) {
13991
- await deliverBuffered(buffered.payload);
13992
- } else {
13993
- await this.store.putSignalWaiter({
13994
- token: cmd.signal,
13995
- runId: run.id,
13996
- seq: cmd.seq
13997
- });
13998
- const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
13999
- if (lateBuffered) {
14000
- await this.store.removeSignalWaiter({
14001
- token: cmd.signal,
14002
- runId: run.id,
14003
- seq: cmd.seq
14004
- });
14005
- await deliverBuffered(lateBuffered.payload);
14006
- }
14007
- }
14089
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
14008
14090
  } else if (cmd.kind === "waitAny") {
14009
14091
  const state = await enterWaitAny({
14010
14092
  store: this.store,
@@ -14036,6 +14118,100 @@ ${run.workflowVersion}`;
14036
14118
  } else {
14037
14119
  this.redeliverFinishedChild(childId, existingChild);
14038
14120
  }
14121
+ } else if (cmd.kind === "sleepUntil") {
14122
+ const existing = await this.store.getCheckpoint(run.id, cmd.seq);
14123
+ const deadline = existing?.wakeAt ?? cmd.until;
14124
+ if (!existing) {
14125
+ await this.store.saveCheckpoint(stepCheckpoint({
14126
+ runId: run.id,
14127
+ seq: cmd.seq,
14128
+ name: `sleep:${cmd.seq}`,
14129
+ kind: "sleep",
14130
+ status: "pending",
14131
+ attempts: 1,
14132
+ wakeAt: deadline,
14133
+ enqueuedAt: at,
14134
+ startedAt: at,
14135
+ finishedAt: at
14136
+ }));
14137
+ }
14138
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14139
+ } else if (cmd.kind === "waitSignalTimeout") {
14140
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14141
+ const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
14142
+ waitSeq: cmd.seq,
14143
+ token: cmd.signal
14144
+ });
14145
+ if (deadline <= this.clock()) {
14146
+ this.resumeSoon(run.id);
14147
+ continue;
14148
+ }
14149
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
14150
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14151
+ } else if (cmd.kind === "waitEvent") {
14152
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14153
+ const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
14154
+ let deadline;
14155
+ if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
14156
+ deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
14157
+ waitSeq: cmd.seq,
14158
+ token
14159
+ });
14160
+ if (deadline <= this.clock()) {
14161
+ this.resumeSoon(run.id);
14162
+ continue;
14163
+ }
14164
+ }
14165
+ await this.store.putSignalWaiter({
14166
+ token,
14167
+ runId: run.id,
14168
+ seq: cmd.seq
14169
+ });
14170
+ const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
14171
+ if (hit) {
14172
+ await this.store.removeSignalWaiter({
14173
+ token,
14174
+ runId: run.id,
14175
+ seq: cmd.seq
14176
+ });
14177
+ await this.store.saveCheckpoint(instantCheckpoint({
14178
+ runId: run.id,
14179
+ seq: cmd.seq,
14180
+ name: `signal:${token}`,
14181
+ kind: "signal",
14182
+ output: hit.payload
14183
+ }));
14184
+ this.resumeSoon(run.id);
14185
+ } else if (deadline != null) {
14186
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14187
+ }
14188
+ } else if (cmd.kind === "spawnChild") {
14189
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14190
+ if (!await this.store.getRun(cmd.childId)) {
14191
+ try {
14192
+ await this.start(cmd.workflow, cmd.input, cmd.childId, {
14193
+ namespace: run.namespace,
14194
+ ...inheritTenant(run),
14195
+ ...cmd.priority !== void 0 ? {
14196
+ priority: cmd.priority
14197
+ } : {}
14198
+ });
14199
+ } catch (err) {
14200
+ const message = err instanceof Error ? err.message : String(err);
14201
+ this.notifyParent(cmd.childId, {
14202
+ ok: false,
14203
+ error: `child workflow "${cmd.workflow}" failed to start: ${message}`
14204
+ });
14205
+ }
14206
+ }
14207
+ await this.store.saveCheckpoint(instantCheckpoint({
14208
+ runId: run.id,
14209
+ seq: cmd.seq,
14210
+ name: `spawn:${cmd.childId}`,
14211
+ kind: "local",
14212
+ output: cmd.childId
14213
+ }));
14214
+ } else if (cmd.kind === "continueAsNew") {
14039
14215
  } else {
14040
14216
  throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
14041
14217
  }
@@ -14043,6 +14219,245 @@ ${run.workflowVersion}`;
14043
14219
  return wakeAt;
14044
14220
  }
14045
14221
  /**
14222
+ * Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
14223
+ * the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
14224
+ * (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
14225
+ * re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
14226
+ * bails).
14227
+ */
14228
+ async parkRemoteOnSignal(run, seq, token) {
14229
+ const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
14230
+ await this.store.saveCheckpoint(instantCheckpoint({
14231
+ runId: run.id,
14232
+ seq,
14233
+ name: `signal:${token}`,
14234
+ kind: "signal",
14235
+ output: payload
14236
+ }));
14237
+ this.resumeSoon(run.id);
14238
+ }, "deliverBuffered");
14239
+ const buffered = await this.store.takeBufferedSignal(token);
14240
+ if (buffered) {
14241
+ await deliverBuffered(buffered.payload);
14242
+ } else {
14243
+ await this.store.putSignalWaiter({
14244
+ token,
14245
+ runId: run.id,
14246
+ seq
14247
+ });
14248
+ const lateBuffered = await this.store.takeBufferedSignal(token);
14249
+ if (lateBuffered) {
14250
+ await this.store.removeSignalWaiter({
14251
+ token,
14252
+ runId: run.id,
14253
+ seq
14254
+ });
14255
+ await deliverBuffered(lateBuffered.payload);
14256
+ }
14257
+ }
14258
+ }
14259
+ /** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
14260
+ resumeSoon(runId) {
14261
+ setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
14262
+ }
14263
+ /**
14264
+ * Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
14265
+ * `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
14266
+ * back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
14267
+ * deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
14268
+ */
14269
+ async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
14270
+ const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
14271
+ if (recorded?.wakeAt != null) return recorded.wakeAt;
14272
+ const deadline = this.clock() + timeoutMs;
14273
+ const at = /* @__PURE__ */ new Date();
14274
+ await this.store.saveCheckpoint(stepCheckpoint({
14275
+ runId,
14276
+ seq: deadlineSeq,
14277
+ name: `timeout:${label}`,
14278
+ kind: "sleep",
14279
+ status: "pending",
14280
+ attempts: 1,
14281
+ input: wait,
14282
+ wakeAt: deadline,
14283
+ enqueuedAt: at,
14284
+ startedAt: at,
14285
+ finishedAt: at
14286
+ }));
14287
+ return deadline;
14288
+ }
14289
+ /**
14290
+ * Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
14291
+ * (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
14292
+ * time out — the in-process wait removes its waiter the same way when it finds the deadline past.
14293
+ * A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
14294
+ * worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
14295
+ */
14296
+ async expireRemoteDeadlines(runId, checkpoints) {
14297
+ const now = this.clock();
14298
+ const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
14299
+ let out = checkpoints;
14300
+ for (const cp of checkpoints) {
14301
+ if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
14302
+ if (cp.wakeAt == null || cp.wakeAt > now) continue;
14303
+ const wait = cp.input;
14304
+ if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
14305
+ if (settled.has(wait.waitSeq)) continue;
14306
+ await this.store.removeSignalWaiter({
14307
+ token: wait.token,
14308
+ runId,
14309
+ seq: wait.waitSeq
14310
+ });
14311
+ const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
14312
+ if (landed && landed.status === "completed") {
14313
+ out = [
14314
+ ...out.filter((c) => c.seq !== landed.seq),
14315
+ landed
14316
+ ].sort((a, b) => a.seq - b.seq);
14317
+ }
14318
+ }
14319
+ return out;
14320
+ }
14321
+ /**
14322
+ * Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
14323
+ * see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
14324
+ * candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
14325
+ * {@link eventBufferTtlMs} is set, like the reconcile sweep.
14326
+ */
14327
+ async claimBufferedEvent(run, name, match) {
14328
+ const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
14329
+ const nowMs = this.clock();
14330
+ for (const candidate of candidates) {
14331
+ if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
14332
+ await this.store.removeBufferedEvent(candidate.id);
14333
+ continue;
14334
+ }
14335
+ const { payload, tenantIds } = readBufferedEvent(candidate.payload);
14336
+ if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
14337
+ if (!eventMatches(payload, match)) continue;
14338
+ if (!await this.store.removeBufferedEvent(candidate.id)) continue;
14339
+ return {
14340
+ payload
14341
+ };
14342
+ }
14343
+ return void 0;
14344
+ }
14345
+ /** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
14346
+ * sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
14347
+ callLease(policy) {
14348
+ if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
14349
+ return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14350
+ }
14351
+ /**
14352
+ * A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
14353
+ * checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
14354
+ * callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
14355
+ * behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
14356
+ * timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
14357
+ * and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
14358
+ *
14359
+ * - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
14360
+ * the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
14361
+ * dispatched again under the same `stepId` with `attempts + 1`.
14362
+ * - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
14363
+ * re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
14364
+ * workflow can catch, as `RemoteStepTimeout` would surface in-process.
14365
+ */
14366
+ async applyCallPolicy(run, cmd, cp) {
14367
+ const policy = cmd.policy;
14368
+ if (cp.kind !== "remote") return null;
14369
+ if (cp.status === "failed") {
14370
+ if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
14371
+ let due = cp.wakeAt;
14372
+ if (due == null) {
14373
+ due = this.clock() + backoffDelay(cp.attempts, policy);
14374
+ if (due > this.clock()) {
14375
+ await this.store.saveCheckpoint({
14376
+ ...cp,
14377
+ wakeAt: due
14378
+ });
14379
+ return due;
14380
+ }
14381
+ }
14382
+ if (this.clock() < due) return due;
14383
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
14384
+ }
14385
+ if (cp.status !== "pending" || !policy.timeoutMs) return null;
14386
+ if (cp.wakeAt == null) {
14387
+ const lease = this.clock() + policy.timeoutMs;
14388
+ await this.store.saveCheckpoint({
14389
+ ...cp,
14390
+ wakeAt: lease
14391
+ });
14392
+ return this.stepLeaseWakeAt(lease);
14393
+ }
14394
+ if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
14395
+ if (cp.attempts < remoteCallMaxAttempts(policy)) {
14396
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
14397
+ }
14398
+ const id = cp.stepId ?? stepId(run.id, cp.seq);
14399
+ const error2 = {
14400
+ message: new RemoteStepTimeout(id, policy.timeoutMs).message,
14401
+ code: "remote_step_timeout"
14402
+ };
14403
+ await this.store.saveCheckpoint({
14404
+ ...cp,
14405
+ status: "failed",
14406
+ error: error2,
14407
+ wakeAt: void 0,
14408
+ finishedAt: /* @__PURE__ */ new Date()
14409
+ });
14410
+ this.emit({
14411
+ type: "step.failed",
14412
+ runId: run.id,
14413
+ seq: cp.seq,
14414
+ name: cp.name,
14415
+ kind: "remote",
14416
+ error: error2
14417
+ });
14418
+ return void 0;
14419
+ }
14420
+ /** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
14421
+ async redispatchCall(run, cmd, cp, attempt, policy, why) {
14422
+ const lease = this.callLease(policy);
14423
+ const at = /* @__PURE__ */ new Date();
14424
+ await this.store.saveCheckpoint({
14425
+ ...cp,
14426
+ status: "pending",
14427
+ output: void 0,
14428
+ error: void 0,
14429
+ attempts: attempt,
14430
+ wakeAt: lease,
14431
+ enqueuedAt: at,
14432
+ startedAt: at,
14433
+ finishedAt: at,
14434
+ events: why === "timeout" ? [
14435
+ ...cp.events ?? [],
14436
+ lostStepEvent(this.clock(), attempt, "redispatched")
14437
+ ] : cp.events
14438
+ });
14439
+ await this.dispatchRemoteTask({
14440
+ runId: run.id,
14441
+ seq: cp.seq,
14442
+ name: cmd.name,
14443
+ stepId: cp.stepId ?? stepId(run.id, cp.seq),
14444
+ group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
14445
+ input: cp.input,
14446
+ attempt
14447
+ });
14448
+ this.emit({
14449
+ type: "step.started",
14450
+ runId: run.id,
14451
+ seq: cp.seq,
14452
+ name: cmd.name,
14453
+ kind: "remote",
14454
+ ...why === "timeout" ? {
14455
+ redispatched: true
14456
+ } : {}
14457
+ });
14458
+ return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
14459
+ }
14460
+ /**
14046
14461
  * Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
14047
14462
  * exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
14048
14463
  * suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
@@ -18443,6 +18858,7 @@ export {
18443
18858
  PARENT_CLOSE_TAG_PREFIX,
18444
18859
  QueueController,
18445
18860
  REMOTE_EVENT,
18861
+ REMOTE_WORKFLOW_FEATURES,
18446
18862
  RUN_HISTORY_FIXTURE_VERSION,
18447
18863
  RUN_VALUE_FACET_LIMIT,
18448
18864
  RUN_VALUE_FACET_SCAN,
@@ -18600,6 +19016,9 @@ export {
18600
19016
  readBufferedEvent,
18601
19017
  readRunsByIds,
18602
19018
  readSearchAttributes,
19019
+ remoteCallMaxAttempts,
19020
+ remoteCallPolicyOf,
19021
+ remoteCallRetries,
18603
19022
  requiredCapabilities,
18604
19023
  resolveConcurrency,
18605
19024
  resolveRouting,