@dudousxd/nestjs-durable-core 0.102.0 → 0.104.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -881,6 +881,14 @@ function isNonEmptyRunScope(scope) {
881
881
  return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
882
882
  }
883
883
  __name(isNonEmptyRunScope, "isNonEmptyRunScope");
884
+ var REMOTE_WORKFLOW_FEATURES = [
885
+ "callPolicy",
886
+ "sleepUntil",
887
+ "waitSignalTimeout",
888
+ "waitEvent",
889
+ "spawnChild",
890
+ "continueAsNew"
891
+ ];
884
892
  var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
885
893
 
886
894
  // src/events.ts
@@ -3069,6 +3077,35 @@ function codeAndDetails(err) {
3069
3077
  }
3070
3078
  __name(codeAndDetails, "codeAndDetails");
3071
3079
 
3080
+ // src/remote-call-policy.ts
3081
+ function remoteCallPolicyOf(config, opts) {
3082
+ const policy = {};
3083
+ const retries = opts?.retries ?? config?.retries;
3084
+ if (retries !== void 0) policy.retries = retries;
3085
+ const backoff = opts?.backoff ?? config?.backoff;
3086
+ if (backoff !== void 0) policy.backoff = backoff;
3087
+ const backoffMs = opts?.backoffMs ?? config?.backoffMs;
3088
+ if (backoffMs !== void 0) policy.backoffMs = backoffMs;
3089
+ const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
3090
+ if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
3091
+ const jitter = opts?.jitter ?? config?.jitter;
3092
+ if (jitter !== void 0) policy.jitter = jitter;
3093
+ const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
3094
+ if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
3095
+ return Object.keys(policy).length > 0 ? policy : void 0;
3096
+ }
3097
+ __name(remoteCallPolicyOf, "remoteCallPolicyOf");
3098
+ function remoteCallMaxAttempts(policy) {
3099
+ return Math.max(1, policy?.retries ?? 1);
3100
+ }
3101
+ __name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
3102
+ function remoteCallRetries(policy, attempts, error2) {
3103
+ if (policy === void 0 || policy.timeoutMs) return false;
3104
+ if (error2?.retryable === false) return false;
3105
+ return attempts < remoteCallMaxAttempts(policy);
3106
+ }
3107
+ __name(remoteCallRetries, "remoteCallRetries");
3108
+
3072
3109
  // src/remote-event.ts
3073
3110
  var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
3074
3111
  function markRemoteEvent(event) {
@@ -3126,7 +3163,12 @@ var RemoteWorkflowExecutor = class {
3126
3163
  } : {},
3127
3164
  group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
3128
3165
  priority: run.priority,
3129
- attempt: 1
3166
+ attempt: 1,
3167
+ // What this control plane applies beyond the original command kinds, so a worker fails a turn
3168
+ // that needs more instead of emitting a command this engine would not understand.
3169
+ features: [
3170
+ ...REMOTE_WORKFLOW_FEATURES
3171
+ ]
3130
3172
  };
3131
3173
  await this.transport.dispatchWorkflowTask(task);
3132
3174
  }
@@ -4566,6 +4608,14 @@ var LEGACY_SCAN_STATUSES = [
4566
4608
  "cancelling"
4567
4609
  ];
4568
4610
  var byAdmissionOrder = /* @__PURE__ */ __name((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id), "byAdmissionOrder");
4611
+ function singletonKeyTag(tags) {
4612
+ if (!tags) return void 0;
4613
+ for (let i = tags.length - 1; i >= 0; i -= 1) {
4614
+ if (tags[i].startsWith("singleton:")) return tags[i];
4615
+ }
4616
+ return void 0;
4617
+ }
4618
+ __name(singletonKeyTag, "singletonKeyTag");
4569
4619
  function withoutSingletonMarkers(tags) {
4570
4620
  return tags?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
4571
4621
  }
@@ -4594,7 +4644,7 @@ function singletonClaimantIds(rows, tag) {
4594
4644
  __name(singletonClaimantIds, "singletonClaimantIds");
4595
4645
  function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
4596
4646
  if (!Number.isInteger(limit) || limit < 1) throw new Error("singleton limit must be a positive integer");
4597
- const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && run.tags?.includes(tag));
4647
+ const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && singletonKeyTag(run.tags) === tag);
4598
4648
  const candidate = active.find((run) => run.id === runId);
4599
4649
  if (!candidate) return null;
4600
4650
  if (candidate.tags?.includes(SINGLETON_ADMITTED_TAG)) return candidate.tags;
@@ -4622,7 +4672,7 @@ async function clearTerminalSingletonAdmission(runId, patch, getRun) {
4622
4672
  __name(clearTerminalSingletonAdmission, "clearTerminalSingletonAdmission");
4623
4673
  function singletonAdmissionDenialPatch(runs, runId, tag, workflow, retryWakeAt) {
4624
4674
  if (retryWakeAt === void 0) return void 0;
4625
- const candidate = runs.find((run) => run.id === runId && run.workflow === workflow && run.tags?.includes(tag) && SINGLETON_ACTIVE_STATUSES.includes(run.status));
4675
+ const candidate = runs.find((run) => run.id === runId && run.workflow === workflow && singletonKeyTag(run.tags) === tag && SINGLETON_ACTIVE_STATUSES.includes(run.status));
4626
4676
  if (!candidate || candidate.status === "cancelling") return void 0;
4627
4677
  return {
4628
4678
  status: "suspended",
@@ -4671,6 +4721,28 @@ function isLockConflict(error2) {
4671
4721
  return false;
4672
4722
  }
4673
4723
  __name(isLockConflict, "isLockConflict");
4724
+ var UnparseableStoredInput = class {
4725
+ static {
4726
+ __name(this, "UnparseableStoredInput");
4727
+ }
4728
+ error;
4729
+ constructor(error2) {
4730
+ this.error = error2;
4731
+ }
4732
+ };
4733
+ async function admissionInput(run, parseInput) {
4734
+ if (!parseInput) return run.input;
4735
+ try {
4736
+ return await parseInput(run.input);
4737
+ } catch (err) {
4738
+ return new UnparseableStoredInput(err instanceof SchemaValidationError && err.code === "invalid_input" ? err : new SchemaValidationError(`workflow "${run.workflow}" input`, [
4739
+ {
4740
+ message: err instanceof Error ? err.message : String(err)
4741
+ }
4742
+ ], "invalid_input"));
4743
+ }
4744
+ }
4745
+ __name(admissionInput, "admissionInput");
4674
4746
  var SingletonGate = class {
4675
4747
  static {
4676
4748
  __name(this, "SingletonGate");
@@ -4707,12 +4779,19 @@ var SingletonGate = class {
4707
4779
  throw new SingletonQueueFullError(workflow, cfg.key(input), cfg.maxQueueDepth);
4708
4780
  }
4709
4781
  }
4710
- /** Preserve durable holders and atomically claim one of the remaining slots. */
4711
- async admit(run, cfg) {
4782
+ /**
4783
+ * Preserve durable holders and atomically claim one of the remaining slots. With `parseInput` (the
4784
+ * workflow's input schema), the key reads the parsed input, the value `run(ctx, input)` gets; a
4785
+ * stored input that no longer parses answers {@link UnparseableStoredInput}, which the caller
4786
+ * must fail the turn with (never run it).
4787
+ */
4788
+ async admit(run, cfg, parseInput) {
4712
4789
  if (!this.deps.store.tryAdmitSingleton) {
4713
4790
  throw new Error("Singleton workflows require StateStore.tryAdmitSingleton for atomic durable admission");
4714
4791
  }
4715
- const admitted = await this.claim(run, cfg);
4792
+ const keyInput = await admissionInput(run, parseInput);
4793
+ if (keyInput instanceof UnparseableStoredInput) return keyInput;
4794
+ const admitted = await this.claim(run, cfg, this.admissionTag(run, cfg, keyInput, parseInput));
4716
4795
  if (admitted && !run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
4717
4796
  run.tags = [
4718
4797
  ...(run.tags ?? []).filter((tag) => tag !== SINGLETON_TRACKED_TAG),
@@ -4722,17 +4801,43 @@ var SingletonGate = class {
4722
4801
  return admitted;
4723
4802
  }
4724
4803
  /**
4804
+ * The exact tag a turn claims under: the key of the parsed input, the tag `start` stamps. A run
4805
+ * queued by an engine that predates `parseInput` was stamped with the key of its RAW input
4806
+ * instead; when the run carries that tag and not the parsed one, it claims under it, so an
4807
+ * upgrade does not strand it. Both are computed from this run's own input, never read off a tag.
4808
+ * Only that case calls the key with the raw input (what the older engine always did), and only
4809
+ * for a run that carries a `singleton:` tag at all; a key that throws on it gets the parsed tag.
4810
+ */
4811
+ admissionTag(run, cfg, keyInput, parseInput) {
4812
+ const parsed = this.tag(cfg, keyInput);
4813
+ const stamped = singletonKeyTag(run.tags);
4814
+ if (!parseInput || stamped === void 0 || stamped === parsed) return parsed;
4815
+ let raw;
4816
+ try {
4817
+ raw = this.tag(cfg, run.input);
4818
+ } catch {
4819
+ return parsed;
4820
+ }
4821
+ return stamped === raw ? raw : parsed;
4822
+ }
4823
+ /**
4725
4824
  * One claim, retried when the database aborts it as a lock conflict. A conflict that outlasts the
4726
4825
  * retries answers "not admitted", so the engine releases this turn's lease, instead of stranding
4727
4826
  * the run until the lease lapses. It writes nothing to the run: this turn's lease may have lapsed
4728
4827
  * during the claim, and another instance may now hold the run, its slot and newer tags. A later
4729
4828
  * sweep re-drives the run: recovery once it is unleased, or its timer if it is parked.
4829
+ *
4830
+ * The claim is under the tag the key computes NOW, the one `start` stamped from the same input
4831
+ * (the schema is deterministic). The store admits a run only when it carries that exact tag, so
4832
+ * a run whose stored tags lack it (a caller tag from before callers' `singleton:` tags were
4833
+ * dropped, a run started before the workflow became a singleton) is never admitted under some
4834
+ * other key; it waits instead.
4730
4835
  */
4731
- async claim(run, cfg) {
4836
+ async claim(run, cfg, tag) {
4732
4837
  const admit = this.deps.store.tryAdmitSingleton;
4733
4838
  for (let attempt = 1; ; attempt += 1) {
4734
4839
  try {
4735
- return await admit.call(this.deps.store, run.id, this.tag(cfg, run.input), run.workflow, cfg.limit ?? 1, this.retryWakeAt());
4840
+ return await admit.call(this.deps.store, run.id, tag, run.workflow, cfg.limit ?? 1, this.retryWakeAt());
4736
4841
  } catch (error2) {
4737
4842
  if (!isLockConflict(error2)) throw error2;
4738
4843
  if (attempt >= ADMISSION_ATTEMPTS) return false;
@@ -4750,7 +4855,7 @@ var SingletonGate = class {
4750
4855
  async wakeNext(settled) {
4751
4856
  const cfg = this.deps.configFor(settled);
4752
4857
  if (!cfg) return;
4753
- const tag = settled.tags?.find((t) => t.startsWith("singleton:"));
4858
+ const tag = singletonKeyTag(settled.tags);
4754
4859
  if (!tag) return;
4755
4860
  const gated = (await this.deps.store.listRuns({
4756
4861
  tag,
@@ -6899,9 +7004,30 @@ function tenantScopedKey(namespace, key) {
6899
7004
  return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
6900
7005
  }
6901
7006
  __name(tenantScopedKey, "tenantScopedKey");
7007
+ function failingBody(error2) {
7008
+ return async () => {
7009
+ throw error2;
7010
+ };
7011
+ }
7012
+ __name(failingBody, "failingBody");
7013
+ function storedInputError(invalid) {
7014
+ return {
7015
+ message: invalid.error.message,
7016
+ code: "invalid_input",
7017
+ retryable: false,
7018
+ details: {
7019
+ issues: invalid.error.issues
7020
+ }
7021
+ };
7022
+ }
7023
+ __name(storedInputError, "storedInputError");
7024
+ function callerTags(tags) {
7025
+ return tags?.filter((tag) => !tag.startsWith(SINGLETON_TAG_PREFIX));
7026
+ }
7027
+ __name(callerTags, "callerTags");
7028
+ var SINGLETON_TAG_PREFIX = "singleton:";
6902
7029
  function singletonKeyOf(run) {
6903
- const tag = run.tags?.find((t) => t.startsWith("singleton:"));
6904
- return tag?.slice("singleton:".length);
7030
+ return singletonKeyTag(run.tags)?.slice("singleton:".length);
6905
7031
  }
6906
7032
  __name(singletonKeyOf, "singletonKeyOf");
6907
7033
  function conventionSingleton(run, announced, wire) {
@@ -7921,6 +8047,7 @@ var WorkflowEngine = class _WorkflowEngine {
7921
8047
  concurrency: opts?.concurrency,
7922
8048
  executionTimeoutMs: opts?.executionTimeout != null ? parseDuration(opts.executionTimeout) : void 0,
7923
8049
  validateInput: opts?.validateInput,
8050
+ parseInput: opts?.parseInput,
7924
8051
  searchAttributesSchema: opts?.searchAttributesSchema,
7925
8052
  onEvent: opts?.onEvent,
7926
8053
  eventBatch: opts?.eventBatch,
@@ -9622,24 +9749,31 @@ var WorkflowEngine = class _WorkflowEngine {
9622
9749
  if (!registered) {
9623
9750
  throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
9624
9751
  }
9625
- if (parkUnroutable && registered.validateInput) {
9752
+ const check = registered;
9753
+ const parseAndValidate = /* @__PURE__ */ __name(async () => {
9754
+ const parsed = check.parseInput ? await check.parseInput(input) : input;
9755
+ await check.validateInput?.(parsed);
9756
+ return parsed;
9757
+ }, "parseAndValidate");
9758
+ let keyInput;
9759
+ if (parkUnroutable && (registered.validateInput || registered.parseInput)) {
9626
9760
  try {
9627
- await registered.validateInput(input);
9761
+ keyInput = await parseAndValidate();
9628
9762
  } catch (err) {
9629
9763
  throw new TenantStartInvalid(err);
9630
9764
  }
9631
9765
  } else {
9632
- await registered.validateInput?.(input);
9766
+ keyInput = await parseAndValidate();
9633
9767
  }
9634
9768
  const prior = await this.store.getRun(runId);
9635
9769
  if (prior) return this.existingStartResult(prior);
9636
- const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency, input);
9770
+ const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency, keyInput);
9637
9771
  if (quota) await assertConcurrency(this.store, name, quota);
9638
9772
  const now = /* @__PURE__ */ new Date();
9639
9773
  const stampedTags = [
9640
- ...opts?.tags ?? [],
9774
+ ...callerTags(opts?.tags) ?? [],
9641
9775
  ...registered.singleton ? [
9642
- this.singletons.tag(registered.singleton, input)
9776
+ this.singletons.tag(registered.singleton, keyInput)
9643
9777
  ] : [],
9644
9778
  ...quota ? [
9645
9779
  concurrencyTag(quota.key)
@@ -9648,9 +9782,9 @@ var WorkflowEngine = class _WorkflowEngine {
9648
9782
  VERSION_UNDECLARED_TAG
9649
9783
  ] : []
9650
9784
  ];
9651
- const tags = mergeTags(registered.tags, stampedTags)?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
9785
+ const tags = mergeTags(callerTags(registered.tags), stampedTags)?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
9652
9786
  if (registered.singleton) {
9653
- await this.singletons.assertCapacity(name, registered.singleton, input);
9787
+ await this.singletons.assertCapacity(name, registered.singleton, keyInput);
9654
9788
  }
9655
9789
  const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
9656
9790
  const delayed = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > this.clock();
@@ -9739,7 +9873,7 @@ var WorkflowEngine = class _WorkflowEngine {
9739
9873
  };
9740
9874
  const now = /* @__PURE__ */ new Date();
9741
9875
  const tags = mergeTags(void 0, [
9742
- ...opts?.tags ?? [],
9876
+ ...callerTags(opts?.tags) ?? [],
9743
9877
  ...singleton ? [
9744
9878
  this.singletons.tag(singleton, input)
9745
9879
  ] : [],
@@ -9919,7 +10053,7 @@ var WorkflowEngine = class _WorkflowEngine {
9919
10053
  namespace: poolNamespace(opts.namespace) ?? this.namespace,
9920
10054
  input,
9921
10055
  // The same derived tags an admitted or parked start carries, so quota-key queries find it.
9922
- tags: mergeTags(opts.tags, [
10056
+ tags: mergeTags(callerTags(opts.tags), [
9923
10057
  ...opts.concurrency ? [
9924
10058
  concurrencyTag(opts.concurrency.key)
9925
10059
  ] : [],
@@ -13446,14 +13580,23 @@ ${run.workflowVersion}`;
13446
13580
  ...await this.sourceTraceOf(run)
13447
13581
  });
13448
13582
  }
13449
- if (registered.singleton && !await this.singletons.admit(run, registered.singleton)) {
13450
- return this.singletonWaitResult(run);
13583
+ if (registered.singleton) {
13584
+ const admission = await this.singletons.admit(run, registered.singleton, registered.parseInput);
13585
+ if (admission instanceof UnparseableStoredInput) {
13586
+ const result = await this.settleRun(run, {
13587
+ kind: "failed",
13588
+ error: storedInputError(admission)
13589
+ });
13590
+ await this.releaseLeaseBestEffort(run.id);
13591
+ return result;
13592
+ }
13593
+ if (!admission) return this.singletonWaitResult(run);
13451
13594
  }
13452
13595
  const wfToken = tenantGroup(sanitizeQueueToken(remote.group), void 0);
13453
13596
  let wfPlan = await this.planRoute(wfToken, registered.requires);
13454
13597
  if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
13455
13598
  if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
13456
- const checkpoints = await this.replayCheckpoints(run.id);
13599
+ const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
13457
13600
  await this.deliverFinishedChildren(run.id, checkpoints);
13458
13601
  await this.deliverFinishedChildWaiters(run.id, checkpoints);
13459
13602
  const handoff = liveHandoff(run.id, checkpoints);
@@ -13539,6 +13682,10 @@ ${run.workflowVersion}`;
13539
13682
  */
13540
13683
  async applyDecision(run, decision) {
13541
13684
  if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
13685
+ const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
13686
+ if (handoff !== void 0 && decision.status === "completed") {
13687
+ return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
13688
+ }
13542
13689
  if (decision.status === "completed") {
13543
13690
  await this.applyCommands(run, decision.commands);
13544
13691
  return this.settleRun(run, {
@@ -13585,6 +13732,63 @@ ${run.workflowVersion}`;
13585
13732
  return suspended;
13586
13733
  }
13587
13734
  /**
13735
+ * A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
13736
+ * run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
13737
+ * engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
13738
+ * turn replayed stays cancelled and starts nothing.
13739
+ */
13740
+ async applyRemoteContinueAsNew(run, commands, input) {
13741
+ await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
13742
+ const fresh = await this.store.getRun(run.id);
13743
+ if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
13744
+ return {
13745
+ runId: run.id,
13746
+ status: fresh.status,
13747
+ output: fresh.output,
13748
+ error: fresh.error
13749
+ };
13750
+ }
13751
+ const nextId = nextContinuationId(run.id);
13752
+ try {
13753
+ await this.start(run.workflow, input, nextId, {
13754
+ namespace: run.namespace,
13755
+ ...inheritTenant(run)
13756
+ });
13757
+ } catch (err) {
13758
+ const message = err instanceof Error ? err.message : String(err);
13759
+ return this.settleRun(run, {
13760
+ kind: "failed",
13761
+ error: {
13762
+ message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
13763
+ code: "continue_as_new_failed"
13764
+ }
13765
+ });
13766
+ }
13767
+ await this.store.updateRun(run.id, {
13768
+ status: "completed",
13769
+ output: void 0,
13770
+ error: void 0,
13771
+ wakeAt: void 0,
13772
+ updatedAt: /* @__PURE__ */ new Date()
13773
+ });
13774
+ this.emit({
13775
+ type: "run.completed",
13776
+ runId: run.id,
13777
+ workflow: run.workflow,
13778
+ namespace: run.namespace
13779
+ }, {
13780
+ lifecycle: false
13781
+ });
13782
+ void this.notifyParent(run.id, {
13783
+ ok: true,
13784
+ value: void 0
13785
+ });
13786
+ return {
13787
+ runId: run.id,
13788
+ status: "completed"
13789
+ };
13790
+ }
13791
+ /**
13588
13792
  * Re-drive a run whose turn was computed from a history OLDER than the store's.
13589
13793
  *
13590
13794
  * A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
@@ -13602,11 +13806,12 @@ ${run.workflowVersion}`;
13602
13806
  * Scheduled off the current stack so it runs after the caller released the run lease.
13603
13807
  */
13604
13808
  async redriveIfTurnSawStaleHistory(runId, commands) {
13605
- const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
13809
+ const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
13606
13810
  if (blocking.length === 0) return;
13607
13811
  for (const cmd of blocking) {
13608
13812
  const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
13609
- if (checkpoint && checkpoint.status !== "pending") {
13813
+ const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
13814
+ if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
13610
13815
  setTimeout(() => {
13611
13816
  void this.resumeImplicit(runId).catch(() => void 0);
13612
13817
  }, 0).unref?.();
@@ -13855,7 +14060,12 @@ ${run.workflowVersion}`;
13855
14060
  // name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
13856
14061
  name: cp.kind === "signal" ? void 0 : cp.name,
13857
14062
  output: cp.status === "completed" ? cp.output : void 0,
13858
- error: cp.status === "failed" ? cp.error : void 0
14063
+ error: cp.status === "failed" ? cp.error : void 0,
14064
+ // How many times a dispatched step ran: a worker applying a `call` policy decides a retry
14065
+ // from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
14066
+ ...cp.kind === "remote" ? {
14067
+ attempts: cp.attempts
14068
+ } : {}
13859
14069
  });
13860
14070
  } else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
13861
14071
  events.push({
@@ -13912,11 +14122,12 @@ ${run.workflowVersion}`;
13912
14122
  } else if (cmd.kind === "call") {
13913
14123
  const existing = await this.store.getCheckpoint(run.id, cmd.seq);
13914
14124
  if (existing) {
13915
- const leaseAt = redriveLost ? await this.redriveLostCall(run, existing) : void 0;
14125
+ const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
14126
+ const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
13916
14127
  if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
13917
14128
  continue;
13918
14129
  }
13919
- const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14130
+ const lease = this.callLease(cmd.policy);
13920
14131
  await this.store.saveCheckpoint(stepCheckpoint({
13921
14132
  runId: run.id,
13922
14133
  seq: cmd.seq,
@@ -13976,35 +14187,7 @@ ${run.workflowVersion}`;
13976
14187
  }));
13977
14188
  wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
13978
14189
  } else if (cmd.kind === "waitSignal") {
13979
- const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
13980
- await this.store.saveCheckpoint(instantCheckpoint({
13981
- runId: run.id,
13982
- seq: cmd.seq,
13983
- name: `signal:${cmd.signal}`,
13984
- kind: "signal",
13985
- output: payload
13986
- }));
13987
- setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
13988
- }, "deliverBuffered");
13989
- const buffered = await this.store.takeBufferedSignal(cmd.signal);
13990
- if (buffered) {
13991
- await deliverBuffered(buffered.payload);
13992
- } else {
13993
- await this.store.putSignalWaiter({
13994
- token: cmd.signal,
13995
- runId: run.id,
13996
- seq: cmd.seq
13997
- });
13998
- const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
13999
- if (lateBuffered) {
14000
- await this.store.removeSignalWaiter({
14001
- token: cmd.signal,
14002
- runId: run.id,
14003
- seq: cmd.seq
14004
- });
14005
- await deliverBuffered(lateBuffered.payload);
14006
- }
14007
- }
14190
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
14008
14191
  } else if (cmd.kind === "waitAny") {
14009
14192
  const state = await enterWaitAny({
14010
14193
  store: this.store,
@@ -14036,6 +14219,100 @@ ${run.workflowVersion}`;
14036
14219
  } else {
14037
14220
  this.redeliverFinishedChild(childId, existingChild);
14038
14221
  }
14222
+ } else if (cmd.kind === "sleepUntil") {
14223
+ const existing = await this.store.getCheckpoint(run.id, cmd.seq);
14224
+ const deadline = existing?.wakeAt ?? cmd.until;
14225
+ if (!existing) {
14226
+ await this.store.saveCheckpoint(stepCheckpoint({
14227
+ runId: run.id,
14228
+ seq: cmd.seq,
14229
+ name: `sleep:${cmd.seq}`,
14230
+ kind: "sleep",
14231
+ status: "pending",
14232
+ attempts: 1,
14233
+ wakeAt: deadline,
14234
+ enqueuedAt: at,
14235
+ startedAt: at,
14236
+ finishedAt: at
14237
+ }));
14238
+ }
14239
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14240
+ } else if (cmd.kind === "waitSignalTimeout") {
14241
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14242
+ const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
14243
+ waitSeq: cmd.seq,
14244
+ token: cmd.signal
14245
+ });
14246
+ if (deadline <= this.clock()) {
14247
+ this.resumeSoon(run.id);
14248
+ continue;
14249
+ }
14250
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
14251
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14252
+ } else if (cmd.kind === "waitEvent") {
14253
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14254
+ const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
14255
+ let deadline;
14256
+ if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
14257
+ deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
14258
+ waitSeq: cmd.seq,
14259
+ token
14260
+ });
14261
+ if (deadline <= this.clock()) {
14262
+ this.resumeSoon(run.id);
14263
+ continue;
14264
+ }
14265
+ }
14266
+ await this.store.putSignalWaiter({
14267
+ token,
14268
+ runId: run.id,
14269
+ seq: cmd.seq
14270
+ });
14271
+ const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
14272
+ if (hit) {
14273
+ await this.store.removeSignalWaiter({
14274
+ token,
14275
+ runId: run.id,
14276
+ seq: cmd.seq
14277
+ });
14278
+ await this.store.saveCheckpoint(instantCheckpoint({
14279
+ runId: run.id,
14280
+ seq: cmd.seq,
14281
+ name: `signal:${token}`,
14282
+ kind: "signal",
14283
+ output: hit.payload
14284
+ }));
14285
+ this.resumeSoon(run.id);
14286
+ } else if (deadline != null) {
14287
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14288
+ }
14289
+ } else if (cmd.kind === "spawnChild") {
14290
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14291
+ if (!await this.store.getRun(cmd.childId)) {
14292
+ try {
14293
+ await this.start(cmd.workflow, cmd.input, cmd.childId, {
14294
+ namespace: run.namespace,
14295
+ ...inheritTenant(run),
14296
+ ...cmd.priority !== void 0 ? {
14297
+ priority: cmd.priority
14298
+ } : {}
14299
+ });
14300
+ } catch (err) {
14301
+ const message = err instanceof Error ? err.message : String(err);
14302
+ this.notifyParent(cmd.childId, {
14303
+ ok: false,
14304
+ error: `child workflow "${cmd.workflow}" failed to start: ${message}`
14305
+ });
14306
+ }
14307
+ }
14308
+ await this.store.saveCheckpoint(instantCheckpoint({
14309
+ runId: run.id,
14310
+ seq: cmd.seq,
14311
+ name: `spawn:${cmd.childId}`,
14312
+ kind: "local",
14313
+ output: cmd.childId
14314
+ }));
14315
+ } else if (cmd.kind === "continueAsNew") {
14039
14316
  } else {
14040
14317
  throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
14041
14318
  }
@@ -14043,6 +14320,245 @@ ${run.workflowVersion}`;
14043
14320
  return wakeAt;
14044
14321
  }
14045
14322
  /**
14323
+ * Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
14324
+ * the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
14325
+ * (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
14326
+ * re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
14327
+ * bails).
14328
+ */
14329
+ async parkRemoteOnSignal(run, seq, token) {
14330
+ const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
14331
+ await this.store.saveCheckpoint(instantCheckpoint({
14332
+ runId: run.id,
14333
+ seq,
14334
+ name: `signal:${token}`,
14335
+ kind: "signal",
14336
+ output: payload
14337
+ }));
14338
+ this.resumeSoon(run.id);
14339
+ }, "deliverBuffered");
14340
+ const buffered = await this.store.takeBufferedSignal(token);
14341
+ if (buffered) {
14342
+ await deliverBuffered(buffered.payload);
14343
+ } else {
14344
+ await this.store.putSignalWaiter({
14345
+ token,
14346
+ runId: run.id,
14347
+ seq
14348
+ });
14349
+ const lateBuffered = await this.store.takeBufferedSignal(token);
14350
+ if (lateBuffered) {
14351
+ await this.store.removeSignalWaiter({
14352
+ token,
14353
+ runId: run.id,
14354
+ seq
14355
+ });
14356
+ await deliverBuffered(lateBuffered.payload);
14357
+ }
14358
+ }
14359
+ }
14360
+ /** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
14361
+ resumeSoon(runId) {
14362
+ setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
14363
+ }
14364
+ /**
14365
+ * Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
14366
+ * `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
14367
+ * back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
14368
+ * deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
14369
+ */
14370
+ async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
14371
+ const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
14372
+ if (recorded?.wakeAt != null) return recorded.wakeAt;
14373
+ const deadline = this.clock() + timeoutMs;
14374
+ const at = /* @__PURE__ */ new Date();
14375
+ await this.store.saveCheckpoint(stepCheckpoint({
14376
+ runId,
14377
+ seq: deadlineSeq,
14378
+ name: `timeout:${label}`,
14379
+ kind: "sleep",
14380
+ status: "pending",
14381
+ attempts: 1,
14382
+ input: wait,
14383
+ wakeAt: deadline,
14384
+ enqueuedAt: at,
14385
+ startedAt: at,
14386
+ finishedAt: at
14387
+ }));
14388
+ return deadline;
14389
+ }
14390
+ /**
14391
+ * Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
14392
+ * (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
14393
+ * time out — the in-process wait removes its waiter the same way when it finds the deadline past.
14394
+ * A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
14395
+ * worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
14396
+ */
14397
+ async expireRemoteDeadlines(runId, checkpoints) {
14398
+ const now = this.clock();
14399
+ const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
14400
+ let out = checkpoints;
14401
+ for (const cp of checkpoints) {
14402
+ if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
14403
+ if (cp.wakeAt == null || cp.wakeAt > now) continue;
14404
+ const wait = cp.input;
14405
+ if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
14406
+ if (settled.has(wait.waitSeq)) continue;
14407
+ await this.store.removeSignalWaiter({
14408
+ token: wait.token,
14409
+ runId,
14410
+ seq: wait.waitSeq
14411
+ });
14412
+ const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
14413
+ if (landed && landed.status === "completed") {
14414
+ out = [
14415
+ ...out.filter((c) => c.seq !== landed.seq),
14416
+ landed
14417
+ ].sort((a, b) => a.seq - b.seq);
14418
+ }
14419
+ }
14420
+ return out;
14421
+ }
14422
+ /**
14423
+ * Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
14424
+ * see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
14425
+ * candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
14426
+ * {@link eventBufferTtlMs} is set, like the reconcile sweep.
14427
+ */
14428
+ async claimBufferedEvent(run, name, match) {
14429
+ const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
14430
+ const nowMs = this.clock();
14431
+ for (const candidate of candidates) {
14432
+ if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
14433
+ await this.store.removeBufferedEvent(candidate.id);
14434
+ continue;
14435
+ }
14436
+ const { payload, tenantIds } = readBufferedEvent(candidate.payload);
14437
+ if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
14438
+ if (!eventMatches(payload, match)) continue;
14439
+ if (!await this.store.removeBufferedEvent(candidate.id)) continue;
14440
+ return {
14441
+ payload
14442
+ };
14443
+ }
14444
+ return void 0;
14445
+ }
14446
+ /** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
14447
+ * sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
14448
+ callLease(policy) {
14449
+ if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
14450
+ return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14451
+ }
14452
+ /**
14453
+ * A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
14454
+ * checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
14455
+ * callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
14456
+ * behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
14457
+ * timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
14458
+ * and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
14459
+ *
14460
+ * - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
14461
+ * the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
14462
+ * dispatched again under the same `stepId` with `attempts + 1`.
14463
+ * - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
14464
+ * re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
14465
+ * workflow can catch, as `RemoteStepTimeout` would surface in-process.
14466
+ */
14467
+ async applyCallPolicy(run, cmd, cp) {
14468
+ const policy = cmd.policy;
14469
+ if (cp.kind !== "remote") return null;
14470
+ if (cp.status === "failed") {
14471
+ if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
14472
+ let due = cp.wakeAt;
14473
+ if (due == null) {
14474
+ due = this.clock() + backoffDelay(cp.attempts, policy);
14475
+ if (due > this.clock()) {
14476
+ await this.store.saveCheckpoint({
14477
+ ...cp,
14478
+ wakeAt: due
14479
+ });
14480
+ return due;
14481
+ }
14482
+ }
14483
+ if (this.clock() < due) return due;
14484
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
14485
+ }
14486
+ if (cp.status !== "pending" || !policy.timeoutMs) return null;
14487
+ if (cp.wakeAt == null) {
14488
+ const lease = this.clock() + policy.timeoutMs;
14489
+ await this.store.saveCheckpoint({
14490
+ ...cp,
14491
+ wakeAt: lease
14492
+ });
14493
+ return this.stepLeaseWakeAt(lease);
14494
+ }
14495
+ if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
14496
+ if (cp.attempts < remoteCallMaxAttempts(policy)) {
14497
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
14498
+ }
14499
+ const id = cp.stepId ?? stepId(run.id, cp.seq);
14500
+ const error2 = {
14501
+ message: new RemoteStepTimeout(id, policy.timeoutMs).message,
14502
+ code: "remote_step_timeout"
14503
+ };
14504
+ await this.store.saveCheckpoint({
14505
+ ...cp,
14506
+ status: "failed",
14507
+ error: error2,
14508
+ wakeAt: void 0,
14509
+ finishedAt: /* @__PURE__ */ new Date()
14510
+ });
14511
+ this.emit({
14512
+ type: "step.failed",
14513
+ runId: run.id,
14514
+ seq: cp.seq,
14515
+ name: cp.name,
14516
+ kind: "remote",
14517
+ error: error2
14518
+ });
14519
+ return void 0;
14520
+ }
14521
+ /** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
14522
+ async redispatchCall(run, cmd, cp, attempt, policy, why) {
14523
+ const lease = this.callLease(policy);
14524
+ const at = /* @__PURE__ */ new Date();
14525
+ await this.store.saveCheckpoint({
14526
+ ...cp,
14527
+ status: "pending",
14528
+ output: void 0,
14529
+ error: void 0,
14530
+ attempts: attempt,
14531
+ wakeAt: lease,
14532
+ enqueuedAt: at,
14533
+ startedAt: at,
14534
+ finishedAt: at,
14535
+ events: why === "timeout" ? [
14536
+ ...cp.events ?? [],
14537
+ lostStepEvent(this.clock(), attempt, "redispatched")
14538
+ ] : cp.events
14539
+ });
14540
+ await this.dispatchRemoteTask({
14541
+ runId: run.id,
14542
+ seq: cp.seq,
14543
+ name: cmd.name,
14544
+ stepId: cp.stepId ?? stepId(run.id, cp.seq),
14545
+ group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
14546
+ input: cp.input,
14547
+ attempt
14548
+ });
14549
+ this.emit({
14550
+ type: "step.started",
14551
+ runId: run.id,
14552
+ seq: cp.seq,
14553
+ name: cmd.name,
14554
+ kind: "remote",
14555
+ ...why === "timeout" ? {
14556
+ redispatched: true
14557
+ } : {}
14558
+ });
14559
+ return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
14560
+ }
14561
+ /**
14046
14562
  * Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
14047
14563
  * exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
14048
14564
  * suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
@@ -14111,7 +14627,8 @@ ${run.workflowVersion}`;
14111
14627
  attempt: void 0
14112
14628
  }, () => this.runExecutionTurn(run, fn, registered));
14113
14629
  }
14114
- async runExecutionTurn(run, fn, registered) {
14630
+ async runExecutionTurn(run, registeredFn, registered) {
14631
+ let fn = registeredFn;
14115
14632
  if (run.status === "cancelling") this.cancelRequested.add(run.id);
14116
14633
  if (run.status === "pending") {
14117
14634
  if (!await this.markStarted(run, registered?.singleton !== void 0)) {
@@ -14127,8 +14644,10 @@ ${run.workflowVersion}`;
14127
14644
  });
14128
14645
  }
14129
14646
  await this.noteServedWakes(run.id);
14130
- if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
14131
- return this.singletonWaitResult(run);
14647
+ if (run.status !== "cancelling" && registered?.singleton) {
14648
+ const admission = await this.singletons.admit(run, registered.singleton, registered.parseInput);
14649
+ if (admission instanceof UnparseableStoredInput) fn = failingBody(admission.error);
14650
+ else if (!admission) return this.singletonWaitResult(run);
14132
14651
  }
14133
14652
  const stickyOk = this.stickyEligible(run);
14134
14653
  let cont = stickyOk ? this.takeContinuation(run, fn) : void 0;
@@ -18443,6 +18962,7 @@ export {
18443
18962
  PARENT_CLOSE_TAG_PREFIX,
18444
18963
  QueueController,
18445
18964
  REMOTE_EVENT,
18965
+ REMOTE_WORKFLOW_FEATURES,
18446
18966
  RUN_HISTORY_FIXTURE_VERSION,
18447
18967
  RUN_VALUE_FACET_LIMIT,
18448
18968
  RUN_VALUE_FACET_SCAN,
@@ -18600,6 +19120,9 @@ export {
18600
19120
  readBufferedEvent,
18601
19121
  readRunsByIds,
18602
19122
  readSearchAttributes,
19123
+ remoteCallMaxAttempts,
19124
+ remoteCallPolicyOf,
19125
+ remoteCallRetries,
18603
19126
  requiredCapabilities,
18604
19127
  resolveConcurrency,
18605
19128
  resolveRouting,
@@ -18636,6 +19159,7 @@ export {
18636
19159
  singletonAdmissionDenialPatch,
18637
19160
  singletonAdmissionTags,
18638
19161
  singletonClaimantIds,
19162
+ singletonKeyTag,
18639
19163
  sqlComparator,
18640
19164
  stepConfigOf,
18641
19165
  stepId,