@dudousxd/nestjs-durable-core 0.102.0 → 0.103.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -77,6 +77,7 @@ __export(index_exports, {
77
77
  PARENT_CLOSE_TAG_PREFIX: () => PARENT_CLOSE_TAG_PREFIX,
78
78
  QueueController: () => QueueController,
79
79
  REMOTE_EVENT: () => REMOTE_EVENT,
80
+ REMOTE_WORKFLOW_FEATURES: () => REMOTE_WORKFLOW_FEATURES,
80
81
  RUN_HISTORY_FIXTURE_VERSION: () => RUN_HISTORY_FIXTURE_VERSION,
81
82
  RUN_VALUE_FACET_LIMIT: () => RUN_VALUE_FACET_LIMIT,
82
83
  RUN_VALUE_FACET_SCAN: () => RUN_VALUE_FACET_SCAN,
@@ -234,6 +235,9 @@ __export(index_exports, {
234
235
  readBufferedEvent: () => readBufferedEvent,
235
236
  readRunsByIds: () => readRunsByIds,
236
237
  readSearchAttributes: () => readSearchAttributes,
238
+ remoteCallMaxAttempts: () => remoteCallMaxAttempts,
239
+ remoteCallPolicyOf: () => remoteCallPolicyOf,
240
+ remoteCallRetries: () => remoteCallRetries,
237
241
  requiredCapabilities: () => requiredCapabilities,
238
242
  resolveConcurrency: () => resolveConcurrency,
239
243
  resolveRouting: () => resolveRouting,
@@ -1178,6 +1182,14 @@ function isNonEmptyRunScope(scope) {
1178
1182
  return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
1179
1183
  }
1180
1184
  __name(isNonEmptyRunScope, "isNonEmptyRunScope");
1185
+ var REMOTE_WORKFLOW_FEATURES = [
1186
+ "callPolicy",
1187
+ "sleepUntil",
1188
+ "waitSignalTimeout",
1189
+ "waitEvent",
1190
+ "spawnChild",
1191
+ "continueAsNew"
1192
+ ];
1181
1193
  var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
1182
1194
 
1183
1195
  // src/events.ts
@@ -3366,6 +3378,35 @@ function codeAndDetails(err) {
3366
3378
  }
3367
3379
  __name(codeAndDetails, "codeAndDetails");
3368
3380
 
3381
+ // src/remote-call-policy.ts
3382
+ function remoteCallPolicyOf(config, opts) {
3383
+ const policy = {};
3384
+ const retries = opts?.retries ?? config?.retries;
3385
+ if (retries !== void 0) policy.retries = retries;
3386
+ const backoff = opts?.backoff ?? config?.backoff;
3387
+ if (backoff !== void 0) policy.backoff = backoff;
3388
+ const backoffMs = opts?.backoffMs ?? config?.backoffMs;
3389
+ if (backoffMs !== void 0) policy.backoffMs = backoffMs;
3390
+ const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
3391
+ if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
3392
+ const jitter = opts?.jitter ?? config?.jitter;
3393
+ if (jitter !== void 0) policy.jitter = jitter;
3394
+ const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
3395
+ if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
3396
+ return Object.keys(policy).length > 0 ? policy : void 0;
3397
+ }
3398
+ __name(remoteCallPolicyOf, "remoteCallPolicyOf");
3399
+ function remoteCallMaxAttempts(policy) {
3400
+ return Math.max(1, policy?.retries ?? 1);
3401
+ }
3402
+ __name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
3403
+ function remoteCallRetries(policy, attempts, error2) {
3404
+ if (policy === void 0 || policy.timeoutMs) return false;
3405
+ if (error2?.retryable === false) return false;
3406
+ return attempts < remoteCallMaxAttempts(policy);
3407
+ }
3408
+ __name(remoteCallRetries, "remoteCallRetries");
3409
+
3369
3410
  // src/remote-event.ts
3370
3411
  var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
3371
3412
  function markRemoteEvent(event) {
@@ -3423,7 +3464,12 @@ var RemoteWorkflowExecutor = class {
3423
3464
  } : {},
3424
3465
  group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
3425
3466
  priority: run.priority,
3426
- attempt: 1
3467
+ attempt: 1,
3468
+ // What this control plane applies beyond the original command kinds, so a worker fails a turn
3469
+ // that needs more instead of emitting a command this engine would not understand.
3470
+ features: [
3471
+ ...REMOTE_WORKFLOW_FEATURES
3472
+ ]
3427
3473
  };
3428
3474
  await this.transport.dispatchWorkflowTask(task);
3429
3475
  }
@@ -13750,7 +13796,7 @@ ${run.workflowVersion}`;
13750
13796
  let wfPlan = await this.planRoute(wfToken, registered.requires);
13751
13797
  if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
13752
13798
  if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
13753
- const checkpoints = await this.replayCheckpoints(run.id);
13799
+ const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
13754
13800
  await this.deliverFinishedChildren(run.id, checkpoints);
13755
13801
  await this.deliverFinishedChildWaiters(run.id, checkpoints);
13756
13802
  const handoff = liveHandoff(run.id, checkpoints);
@@ -13836,6 +13882,10 @@ ${run.workflowVersion}`;
13836
13882
  */
13837
13883
  async applyDecision(run, decision) {
13838
13884
  if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
13885
+ const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
13886
+ if (handoff !== void 0 && decision.status === "completed") {
13887
+ return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
13888
+ }
13839
13889
  if (decision.status === "completed") {
13840
13890
  await this.applyCommands(run, decision.commands);
13841
13891
  return this.settleRun(run, {
@@ -13882,6 +13932,63 @@ ${run.workflowVersion}`;
13882
13932
  return suspended;
13883
13933
  }
13884
13934
  /**
13935
+ * A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
13936
+ * run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
13937
+ * engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
13938
+ * turn replayed stays cancelled and starts nothing.
13939
+ */
13940
+ async applyRemoteContinueAsNew(run, commands, input) {
13941
+ await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
13942
+ const fresh = await this.store.getRun(run.id);
13943
+ if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
13944
+ return {
13945
+ runId: run.id,
13946
+ status: fresh.status,
13947
+ output: fresh.output,
13948
+ error: fresh.error
13949
+ };
13950
+ }
13951
+ const nextId = nextContinuationId(run.id);
13952
+ try {
13953
+ await this.start(run.workflow, input, nextId, {
13954
+ namespace: run.namespace,
13955
+ ...inheritTenant(run)
13956
+ });
13957
+ } catch (err) {
13958
+ const message = err instanceof Error ? err.message : String(err);
13959
+ return this.settleRun(run, {
13960
+ kind: "failed",
13961
+ error: {
13962
+ message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
13963
+ code: "continue_as_new_failed"
13964
+ }
13965
+ });
13966
+ }
13967
+ await this.store.updateRun(run.id, {
13968
+ status: "completed",
13969
+ output: void 0,
13970
+ error: void 0,
13971
+ wakeAt: void 0,
13972
+ updatedAt: /* @__PURE__ */ new Date()
13973
+ });
13974
+ this.emit({
13975
+ type: "run.completed",
13976
+ runId: run.id,
13977
+ workflow: run.workflow,
13978
+ namespace: run.namespace
13979
+ }, {
13980
+ lifecycle: false
13981
+ });
13982
+ void this.notifyParent(run.id, {
13983
+ ok: true,
13984
+ value: void 0
13985
+ });
13986
+ return {
13987
+ runId: run.id,
13988
+ status: "completed"
13989
+ };
13990
+ }
13991
+ /**
13885
13992
  * Re-drive a run whose turn was computed from a history OLDER than the store's.
13886
13993
  *
13887
13994
  * A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
@@ -13899,11 +14006,12 @@ ${run.workflowVersion}`;
13899
14006
  * Scheduled off the current stack so it runs after the caller released the run lease.
13900
14007
  */
13901
14008
  async redriveIfTurnSawStaleHistory(runId, commands) {
13902
- const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
14009
+ const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
13903
14010
  if (blocking.length === 0) return;
13904
14011
  for (const cmd of blocking) {
13905
14012
  const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
13906
- if (checkpoint && checkpoint.status !== "pending") {
14013
+ const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
14014
+ if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
13907
14015
  setTimeout(() => {
13908
14016
  void this.resumeImplicit(runId).catch(() => void 0);
13909
14017
  }, 0).unref?.();
@@ -14152,7 +14260,12 @@ ${run.workflowVersion}`;
14152
14260
  // name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
14153
14261
  name: cp.kind === "signal" ? void 0 : cp.name,
14154
14262
  output: cp.status === "completed" ? cp.output : void 0,
14155
- error: cp.status === "failed" ? cp.error : void 0
14263
+ error: cp.status === "failed" ? cp.error : void 0,
14264
+ // How many times a dispatched step ran: a worker applying a `call` policy decides a retry
14265
+ // from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
14266
+ ...cp.kind === "remote" ? {
14267
+ attempts: cp.attempts
14268
+ } : {}
14156
14269
  });
14157
14270
  } else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
14158
14271
  events.push({
@@ -14209,11 +14322,12 @@ ${run.workflowVersion}`;
14209
14322
  } else if (cmd.kind === "call") {
14210
14323
  const existing = await this.store.getCheckpoint(run.id, cmd.seq);
14211
14324
  if (existing) {
14212
- const leaseAt = redriveLost ? await this.redriveLostCall(run, existing) : void 0;
14325
+ const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
14326
+ const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
14213
14327
  if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
14214
14328
  continue;
14215
14329
  }
14216
- const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14330
+ const lease = this.callLease(cmd.policy);
14217
14331
  await this.store.saveCheckpoint(stepCheckpoint({
14218
14332
  runId: run.id,
14219
14333
  seq: cmd.seq,
@@ -14273,35 +14387,7 @@ ${run.workflowVersion}`;
14273
14387
  }));
14274
14388
  wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14275
14389
  } else if (cmd.kind === "waitSignal") {
14276
- const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
14277
- await this.store.saveCheckpoint(instantCheckpoint({
14278
- runId: run.id,
14279
- seq: cmd.seq,
14280
- name: `signal:${cmd.signal}`,
14281
- kind: "signal",
14282
- output: payload
14283
- }));
14284
- setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
14285
- }, "deliverBuffered");
14286
- const buffered = await this.store.takeBufferedSignal(cmd.signal);
14287
- if (buffered) {
14288
- await deliverBuffered(buffered.payload);
14289
- } else {
14290
- await this.store.putSignalWaiter({
14291
- token: cmd.signal,
14292
- runId: run.id,
14293
- seq: cmd.seq
14294
- });
14295
- const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
14296
- if (lateBuffered) {
14297
- await this.store.removeSignalWaiter({
14298
- token: cmd.signal,
14299
- runId: run.id,
14300
- seq: cmd.seq
14301
- });
14302
- await deliverBuffered(lateBuffered.payload);
14303
- }
14304
- }
14390
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
14305
14391
  } else if (cmd.kind === "waitAny") {
14306
14392
  const state = await enterWaitAny({
14307
14393
  store: this.store,
@@ -14333,6 +14419,100 @@ ${run.workflowVersion}`;
14333
14419
  } else {
14334
14420
  this.redeliverFinishedChild(childId, existingChild);
14335
14421
  }
14422
+ } else if (cmd.kind === "sleepUntil") {
14423
+ const existing = await this.store.getCheckpoint(run.id, cmd.seq);
14424
+ const deadline = existing?.wakeAt ?? cmd.until;
14425
+ if (!existing) {
14426
+ await this.store.saveCheckpoint(stepCheckpoint({
14427
+ runId: run.id,
14428
+ seq: cmd.seq,
14429
+ name: `sleep:${cmd.seq}`,
14430
+ kind: "sleep",
14431
+ status: "pending",
14432
+ attempts: 1,
14433
+ wakeAt: deadline,
14434
+ enqueuedAt: at,
14435
+ startedAt: at,
14436
+ finishedAt: at
14437
+ }));
14438
+ }
14439
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14440
+ } else if (cmd.kind === "waitSignalTimeout") {
14441
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14442
+ const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
14443
+ waitSeq: cmd.seq,
14444
+ token: cmd.signal
14445
+ });
14446
+ if (deadline <= this.clock()) {
14447
+ this.resumeSoon(run.id);
14448
+ continue;
14449
+ }
14450
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
14451
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14452
+ } else if (cmd.kind === "waitEvent") {
14453
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14454
+ const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
14455
+ let deadline;
14456
+ if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
14457
+ deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
14458
+ waitSeq: cmd.seq,
14459
+ token
14460
+ });
14461
+ if (deadline <= this.clock()) {
14462
+ this.resumeSoon(run.id);
14463
+ continue;
14464
+ }
14465
+ }
14466
+ await this.store.putSignalWaiter({
14467
+ token,
14468
+ runId: run.id,
14469
+ seq: cmd.seq
14470
+ });
14471
+ const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
14472
+ if (hit) {
14473
+ await this.store.removeSignalWaiter({
14474
+ token,
14475
+ runId: run.id,
14476
+ seq: cmd.seq
14477
+ });
14478
+ await this.store.saveCheckpoint(instantCheckpoint({
14479
+ runId: run.id,
14480
+ seq: cmd.seq,
14481
+ name: `signal:${token}`,
14482
+ kind: "signal",
14483
+ output: hit.payload
14484
+ }));
14485
+ this.resumeSoon(run.id);
14486
+ } else if (deadline != null) {
14487
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14488
+ }
14489
+ } else if (cmd.kind === "spawnChild") {
14490
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14491
+ if (!await this.store.getRun(cmd.childId)) {
14492
+ try {
14493
+ await this.start(cmd.workflow, cmd.input, cmd.childId, {
14494
+ namespace: run.namespace,
14495
+ ...inheritTenant(run),
14496
+ ...cmd.priority !== void 0 ? {
14497
+ priority: cmd.priority
14498
+ } : {}
14499
+ });
14500
+ } catch (err) {
14501
+ const message = err instanceof Error ? err.message : String(err);
14502
+ this.notifyParent(cmd.childId, {
14503
+ ok: false,
14504
+ error: `child workflow "${cmd.workflow}" failed to start: ${message}`
14505
+ });
14506
+ }
14507
+ }
14508
+ await this.store.saveCheckpoint(instantCheckpoint({
14509
+ runId: run.id,
14510
+ seq: cmd.seq,
14511
+ name: `spawn:${cmd.childId}`,
14512
+ kind: "local",
14513
+ output: cmd.childId
14514
+ }));
14515
+ } else if (cmd.kind === "continueAsNew") {
14336
14516
  } else {
14337
14517
  throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
14338
14518
  }
@@ -14340,6 +14520,245 @@ ${run.workflowVersion}`;
14340
14520
  return wakeAt;
14341
14521
  }
14342
14522
  /**
14523
+ * Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
14524
+ * the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
14525
+ * (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
14526
+ * re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
14527
+ * bails).
14528
+ */
14529
+ async parkRemoteOnSignal(run, seq, token) {
14530
+ const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
14531
+ await this.store.saveCheckpoint(instantCheckpoint({
14532
+ runId: run.id,
14533
+ seq,
14534
+ name: `signal:${token}`,
14535
+ kind: "signal",
14536
+ output: payload
14537
+ }));
14538
+ this.resumeSoon(run.id);
14539
+ }, "deliverBuffered");
14540
+ const buffered = await this.store.takeBufferedSignal(token);
14541
+ if (buffered) {
14542
+ await deliverBuffered(buffered.payload);
14543
+ } else {
14544
+ await this.store.putSignalWaiter({
14545
+ token,
14546
+ runId: run.id,
14547
+ seq
14548
+ });
14549
+ const lateBuffered = await this.store.takeBufferedSignal(token);
14550
+ if (lateBuffered) {
14551
+ await this.store.removeSignalWaiter({
14552
+ token,
14553
+ runId: run.id,
14554
+ seq
14555
+ });
14556
+ await deliverBuffered(lateBuffered.payload);
14557
+ }
14558
+ }
14559
+ }
14560
+ /** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
14561
+ resumeSoon(runId) {
14562
+ setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
14563
+ }
14564
+ /**
14565
+ * Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
14566
+ * `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
14567
+ * back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
14568
+ * deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
14569
+ */
14570
+ async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
14571
+ const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
14572
+ if (recorded?.wakeAt != null) return recorded.wakeAt;
14573
+ const deadline = this.clock() + timeoutMs;
14574
+ const at = /* @__PURE__ */ new Date();
14575
+ await this.store.saveCheckpoint(stepCheckpoint({
14576
+ runId,
14577
+ seq: deadlineSeq,
14578
+ name: `timeout:${label}`,
14579
+ kind: "sleep",
14580
+ status: "pending",
14581
+ attempts: 1,
14582
+ input: wait,
14583
+ wakeAt: deadline,
14584
+ enqueuedAt: at,
14585
+ startedAt: at,
14586
+ finishedAt: at
14587
+ }));
14588
+ return deadline;
14589
+ }
14590
+ /**
14591
+ * Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
14592
+ * (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
14593
+ * time out — the in-process wait removes its waiter the same way when it finds the deadline past.
14594
+ * A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
14595
+ * worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
14596
+ */
14597
+ async expireRemoteDeadlines(runId, checkpoints) {
14598
+ const now = this.clock();
14599
+ const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
14600
+ let out = checkpoints;
14601
+ for (const cp of checkpoints) {
14602
+ if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
14603
+ if (cp.wakeAt == null || cp.wakeAt > now) continue;
14604
+ const wait = cp.input;
14605
+ if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
14606
+ if (settled.has(wait.waitSeq)) continue;
14607
+ await this.store.removeSignalWaiter({
14608
+ token: wait.token,
14609
+ runId,
14610
+ seq: wait.waitSeq
14611
+ });
14612
+ const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
14613
+ if (landed && landed.status === "completed") {
14614
+ out = [
14615
+ ...out.filter((c) => c.seq !== landed.seq),
14616
+ landed
14617
+ ].sort((a, b) => a.seq - b.seq);
14618
+ }
14619
+ }
14620
+ return out;
14621
+ }
14622
+ /**
14623
+ * Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
14624
+ * see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
14625
+ * candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
14626
+ * {@link eventBufferTtlMs} is set, like the reconcile sweep.
14627
+ */
14628
+ async claimBufferedEvent(run, name, match) {
14629
+ const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
14630
+ const nowMs = this.clock();
14631
+ for (const candidate of candidates) {
14632
+ if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
14633
+ await this.store.removeBufferedEvent(candidate.id);
14634
+ continue;
14635
+ }
14636
+ const { payload, tenantIds } = readBufferedEvent(candidate.payload);
14637
+ if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
14638
+ if (!eventMatches(payload, match)) continue;
14639
+ if (!await this.store.removeBufferedEvent(candidate.id)) continue;
14640
+ return {
14641
+ payload
14642
+ };
14643
+ }
14644
+ return void 0;
14645
+ }
14646
+ /** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
14647
+ * sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
14648
+ callLease(policy) {
14649
+ if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
14650
+ return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14651
+ }
14652
+ /**
14653
+ * A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
14654
+ * checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
14655
+ * callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
14656
+ * behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
14657
+ * timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
14658
+ * and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
14659
+ *
14660
+ * - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
14661
+ * the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
14662
+ * dispatched again under the same `stepId` with `attempts + 1`.
14663
+ * - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
14664
+ * re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
14665
+ * workflow can catch, as `RemoteStepTimeout` would surface in-process.
14666
+ */
14667
+ async applyCallPolicy(run, cmd, cp) {
14668
+ const policy = cmd.policy;
14669
+ if (cp.kind !== "remote") return null;
14670
+ if (cp.status === "failed") {
14671
+ if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
14672
+ let due = cp.wakeAt;
14673
+ if (due == null) {
14674
+ due = this.clock() + backoffDelay(cp.attempts, policy);
14675
+ if (due > this.clock()) {
14676
+ await this.store.saveCheckpoint({
14677
+ ...cp,
14678
+ wakeAt: due
14679
+ });
14680
+ return due;
14681
+ }
14682
+ }
14683
+ if (this.clock() < due) return due;
14684
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
14685
+ }
14686
+ if (cp.status !== "pending" || !policy.timeoutMs) return null;
14687
+ if (cp.wakeAt == null) {
14688
+ const lease = this.clock() + policy.timeoutMs;
14689
+ await this.store.saveCheckpoint({
14690
+ ...cp,
14691
+ wakeAt: lease
14692
+ });
14693
+ return this.stepLeaseWakeAt(lease);
14694
+ }
14695
+ if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
14696
+ if (cp.attempts < remoteCallMaxAttempts(policy)) {
14697
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
14698
+ }
14699
+ const id = cp.stepId ?? stepId(run.id, cp.seq);
14700
+ const error2 = {
14701
+ message: new RemoteStepTimeout(id, policy.timeoutMs).message,
14702
+ code: "remote_step_timeout"
14703
+ };
14704
+ await this.store.saveCheckpoint({
14705
+ ...cp,
14706
+ status: "failed",
14707
+ error: error2,
14708
+ wakeAt: void 0,
14709
+ finishedAt: /* @__PURE__ */ new Date()
14710
+ });
14711
+ this.emit({
14712
+ type: "step.failed",
14713
+ runId: run.id,
14714
+ seq: cp.seq,
14715
+ name: cp.name,
14716
+ kind: "remote",
14717
+ error: error2
14718
+ });
14719
+ return void 0;
14720
+ }
14721
+ /** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
14722
+ async redispatchCall(run, cmd, cp, attempt, policy, why) {
14723
+ const lease = this.callLease(policy);
14724
+ const at = /* @__PURE__ */ new Date();
14725
+ await this.store.saveCheckpoint({
14726
+ ...cp,
14727
+ status: "pending",
14728
+ output: void 0,
14729
+ error: void 0,
14730
+ attempts: attempt,
14731
+ wakeAt: lease,
14732
+ enqueuedAt: at,
14733
+ startedAt: at,
14734
+ finishedAt: at,
14735
+ events: why === "timeout" ? [
14736
+ ...cp.events ?? [],
14737
+ lostStepEvent(this.clock(), attempt, "redispatched")
14738
+ ] : cp.events
14739
+ });
14740
+ await this.dispatchRemoteTask({
14741
+ runId: run.id,
14742
+ seq: cp.seq,
14743
+ name: cmd.name,
14744
+ stepId: cp.stepId ?? stepId(run.id, cp.seq),
14745
+ group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
14746
+ input: cp.input,
14747
+ attempt
14748
+ });
14749
+ this.emit({
14750
+ type: "step.started",
14751
+ runId: run.id,
14752
+ seq: cp.seq,
14753
+ name: cmd.name,
14754
+ kind: "remote",
14755
+ ...why === "timeout" ? {
14756
+ redispatched: true
14757
+ } : {}
14758
+ });
14759
+ return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
14760
+ }
14761
+ /**
14343
14762
  * Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
14344
14763
  * exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
14345
14764
  * suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
@@ -18741,6 +19160,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
18741
19160
  PARENT_CLOSE_TAG_PREFIX,
18742
19161
  QueueController,
18743
19162
  REMOTE_EVENT,
19163
+ REMOTE_WORKFLOW_FEATURES,
18744
19164
  RUN_HISTORY_FIXTURE_VERSION,
18745
19165
  RUN_VALUE_FACET_LIMIT,
18746
19166
  RUN_VALUE_FACET_SCAN,
@@ -18898,6 +19318,9 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
18898
19318
  readBufferedEvent,
18899
19319
  readRunsByIds,
18900
19320
  readSearchAttributes,
19321
+ remoteCallMaxAttempts,
19322
+ remoteCallPolicyOf,
19323
+ remoteCallRetries,
18901
19324
  requiredCapabilities,
18902
19325
  resolveConcurrency,
18903
19326
  resolveRouting,