@dudousxd/nestjs-durable-core 0.101.0 → 0.103.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -881,6 +881,14 @@ function isNonEmptyRunScope(scope) {
881
881
  return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
882
882
  }
883
883
  __name(isNonEmptyRunScope, "isNonEmptyRunScope");
884
+ var REMOTE_WORKFLOW_FEATURES = [
885
+ "callPolicy",
886
+ "sleepUntil",
887
+ "waitSignalTimeout",
888
+ "waitEvent",
889
+ "spawnChild",
890
+ "continueAsNew"
891
+ ];
884
892
  var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
885
893
 
886
894
  // src/events.ts
@@ -2342,6 +2350,13 @@ function canonicalizeRegistrations(regs) {
2342
2350
  subscribes: [
2343
2351
  ...new Set(r.subscribes)
2344
2352
  ].sort()
2353
+ } : {},
2354
+ // Conditional too: a registration without them keeps its hash; a changed option changes it.
2355
+ ...r.options !== void 0 ? {
2356
+ options: stableJson(r.options)
2357
+ } : {},
2358
+ ...r.deadLetter !== void 0 ? {
2359
+ deadLetter: r.deadLetter
2345
2360
  } : {}
2346
2361
  }));
2347
2362
  const byJson = /* @__PURE__ */ new Map();
@@ -3062,6 +3077,35 @@ function codeAndDetails(err) {
3062
3077
  }
3063
3078
  __name(codeAndDetails, "codeAndDetails");
3064
3079
 
3080
+ // src/remote-call-policy.ts
3081
+ function remoteCallPolicyOf(config, opts) {
3082
+ const policy = {};
3083
+ const retries = opts?.retries ?? config?.retries;
3084
+ if (retries !== void 0) policy.retries = retries;
3085
+ const backoff = opts?.backoff ?? config?.backoff;
3086
+ if (backoff !== void 0) policy.backoff = backoff;
3087
+ const backoffMs = opts?.backoffMs ?? config?.backoffMs;
3088
+ if (backoffMs !== void 0) policy.backoffMs = backoffMs;
3089
+ const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
3090
+ if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
3091
+ const jitter = opts?.jitter ?? config?.jitter;
3092
+ if (jitter !== void 0) policy.jitter = jitter;
3093
+ const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
3094
+ if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
3095
+ return Object.keys(policy).length > 0 ? policy : void 0;
3096
+ }
3097
+ __name(remoteCallPolicyOf, "remoteCallPolicyOf");
3098
+ function remoteCallMaxAttempts(policy) {
3099
+ return Math.max(1, policy?.retries ?? 1);
3100
+ }
3101
+ __name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
3102
+ function remoteCallRetries(policy, attempts, error2) {
3103
+ if (policy === void 0 || policy.timeoutMs) return false;
3104
+ if (error2?.retryable === false) return false;
3105
+ return attempts < remoteCallMaxAttempts(policy);
3106
+ }
3107
+ __name(remoteCallRetries, "remoteCallRetries");
3108
+
3065
3109
  // src/remote-event.ts
3066
3110
  var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
3067
3111
  function markRemoteEvent(event) {
@@ -3119,7 +3163,12 @@ var RemoteWorkflowExecutor = class {
3119
3163
  } : {},
3120
3164
  group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
3121
3165
  priority: run.priority,
3122
- attempt: 1
3166
+ attempt: 1,
3167
+ // What this control plane applies beyond the original command kinds, so a worker fails a turn
3168
+ // that needs more instead of emitting a command this engine would not understand.
3169
+ features: [
3170
+ ...REMOTE_WORKFLOW_FEATURES
3171
+ ]
3123
3172
  };
3124
3173
  await this.transport.dispatchWorkflowTask(task);
3125
3174
  }
@@ -4052,7 +4101,8 @@ var ScheduleClient = class {
4052
4101
  const input = specOf(record).input;
4053
4102
  const opts = startOptions(record);
4054
4103
  if (isAnnounced(record) && this.host.startParked) {
4055
- return this.host.startParked(record.workflow, input, runId, opts);
4104
+ const singleton = record.state.announce?.singleton;
4105
+ return singleton ? this.host.startParked(record.workflow, input, runId, opts, singleton) : this.host.startParked(record.workflow, input, runId, opts);
4056
4106
  }
4057
4107
  return this.host.start(record.workflow, input, runId, opts);
4058
4108
  }
@@ -4248,7 +4298,8 @@ var ScheduleClient = class {
4248
4298
  announce: {
4249
4299
  key: doc.key,
4250
4300
  fingerprint,
4251
- declaredPaused: doc.declaredPaused
4301
+ declaredPaused: doc.declaredPaused,
4302
+ singleton: doc.singleton
4252
4303
  }
4253
4304
  }),
4254
4305
  updatedAt: new Date(now)
@@ -4285,7 +4336,10 @@ var ScheduleClient = class {
4285
4336
  announce: {
4286
4337
  key: item.key,
4287
4338
  fingerprint,
4288
- declaredPaused
4339
+ declaredPaused,
4340
+ ...validSingleton(item.singleton) ? {
4341
+ singleton: item.singleton
4342
+ } : {}
4289
4343
  }
4290
4344
  });
4291
4345
  out.adopted.push(id);
@@ -4435,6 +4489,10 @@ function specOf(record) {
4435
4489
  return record.spec;
4436
4490
  }
4437
4491
  __name(specOf, "specOf");
4492
+ function validSingleton(value) {
4493
+ return value !== null && typeof value === "object" && typeof value.key === "string" && value.key !== "";
4494
+ }
4495
+ __name(validSingleton, "validSingleton");
4438
4496
  function isAnnounced(record) {
4439
4497
  return record.state?.announce !== void 0;
4440
4498
  }
@@ -6878,6 +6936,43 @@ var DEFAULT_STICKY_RUNS = 1e3;
6878
6936
  var DEFAULT_STICKY_IDLE_MS = 3e5;
6879
6937
  var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
6880
6938
  var ANNOUNCED_ROUTES_TTL_MS = 5e3;
6939
+ var ANNOUNCED_TIMEOUTS_TTL_MS = 15e3;
6940
+ function tenantScopedKey(namespace, key) {
6941
+ return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
6942
+ }
6943
+ __name(tenantScopedKey, "tenantScopedKey");
6944
+ function singletonKeyOf(run) {
6945
+ const tag = run.tags?.find((t) => t.startsWith("singleton:"));
6946
+ return tag?.slice("singleton:".length);
6947
+ }
6948
+ __name(singletonKeyOf, "singletonKeyOf");
6949
+ function conventionSingleton(run, announced, wire) {
6950
+ if (wire !== void 0) {
6951
+ const key2 = wire.key;
6952
+ return {
6953
+ key: /* @__PURE__ */ __name(() => key2, "key"),
6954
+ ...typeof wire.limit === "number" ? {
6955
+ limit: wire.limit
6956
+ } : {},
6957
+ ...typeof wire.maxQueueDepth === "number" ? {
6958
+ maxQueueDepth: wire.maxQueueDepth
6959
+ } : {}
6960
+ };
6961
+ }
6962
+ const key = singletonKeyOf(run);
6963
+ if (key === void 0 || announced === void 0) return void 0;
6964
+ if (announced === null) return {
6965
+ key: /* @__PURE__ */ __name(() => key, "key"),
6966
+ limit: 1
6967
+ };
6968
+ const declared = announced.options?.singleton;
6969
+ if (declared === void 0) return void 0;
6970
+ return {
6971
+ key: /* @__PURE__ */ __name(() => key, "key"),
6972
+ limit: typeof declared.limit === "number" ? declared.limit : 1
6973
+ };
6974
+ }
6975
+ __name(conventionSingleton, "conventionSingleton");
6881
6976
  function servedNamespace(namespace, group) {
6882
6977
  if (namespace) return namespace;
6883
6978
  const at = group?.lastIndexOf("@") ?? -1;
@@ -7367,13 +7462,28 @@ var WorkflowEngine = class _WorkflowEngine {
7367
7462
  if (this.queuesRunsOverTransport) void this.wake(runId);
7368
7463
  else this.runDispatcher.dispatch(runId);
7369
7464
  }, "dispatch"),
7370
- configFor: /* @__PURE__ */ __name((run) => (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton, "configFor")
7465
+ configFor: /* @__PURE__ */ __name((run) => {
7466
+ const local = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton;
7467
+ if (local) return local;
7468
+ const key = this.latest.has(run.workflow) ? void 0 : singletonKeyOf(run);
7469
+ if (key === void 0) return void 0;
7470
+ const limit = this.conventionSingletonLimits.get(`${runNamespace(run.namespace)}\0${run.workflow}`);
7471
+ return {
7472
+ key: /* @__PURE__ */ __name(() => key, "key"),
7473
+ ...limit !== void 0 ? {
7474
+ limit
7475
+ } : {}
7476
+ };
7477
+ }, "configFor")
7371
7478
  });
7372
7479
  this.schedules = new ScheduleClient(this.store, {
7373
7480
  start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
7374
7481
  // An announced (tenant) schedule fires through the tenant start path: with no live worker
7375
7482
  // of its app, the run waits `blocked` instead of the window being lost.
7376
- startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true), "startParked"),
7483
+ startParked: /* @__PURE__ */ __name((workflow, input, runId, opts, singleton) => this.startWith(workflow, input, runId, opts, true, singleton ? {
7484
+ ...singleton,
7485
+ key: tenantScopedKey(opts?.namespace, singleton.key)
7486
+ } : void 0), "startParked"),
7377
7487
  getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
7378
7488
  isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
7379
7489
  tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
@@ -8183,6 +8293,39 @@ var WorkflowEngine = class _WorkflowEngine {
8183
8293
  } catch {
8184
8294
  }
8185
8295
  }
8296
+ if (!this.latest.has(run.workflow)) this.inBackground(this.routeAnnouncedDeadLetter(run));
8297
+ }
8298
+ /**
8299
+ * The dead-letter workflow the live workers serving `run`'s workflow announce (a tenant's inline
8300
+ * `@DeadLetter()` handler, its `deadLetterWorkflow`, or its module-level default), or `undefined`
8301
+ * when the workflow is registered on this engine (its own routing applies), they announce none, or
8302
+ * they cannot be read. A dead-letter listener with a catch-all default (`DurableModule`'s
8303
+ * `deadLetterWorkflow`) checks it first, so the app's own route wins over the default.
8304
+ */
8305
+ async announcedDeadLetter(run) {
8306
+ if (this.latest.has(run.workflow) || this.isRegisteredHere(run)) return void 0;
8307
+ const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
8308
+ const reg = await this.announcedRegistration(group, run.workflow, run.workflowVersion);
8309
+ const target = reg?.deadLetter;
8310
+ return typeof target === "string" && target !== "" ? target : void 0;
8311
+ }
8312
+ /**
8313
+ * Start the announced dead-letter workflow of a dead run (see {@link announcedDeadLetter}) in the
8314
+ * run's namespace, as `dlq:<runId>` with the `DeadLetter` payload, through the tenant start path:
8315
+ * with no live worker of the app it waits `blocked` instead of being lost. Idempotent by run id.
8316
+ */
8317
+ async routeAnnouncedDeadLetter(run) {
8318
+ const target = await this.announcedDeadLetter(run);
8319
+ if (!target) return;
8320
+ await this.startWith(target, {
8321
+ deadRunId: run.id,
8322
+ workflow: run.workflow,
8323
+ input: run.input,
8324
+ error: run.error
8325
+ }, `dlq:${run.id}`, {
8326
+ namespace: run.namespace,
8327
+ ...inheritTenant(run)
8328
+ }, true);
8186
8329
  }
8187
8330
  /**
8188
8331
  * Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
@@ -9483,7 +9626,7 @@ var WorkflowEngine = class _WorkflowEngine {
9483
9626
  * the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
9484
9627
  * a start whose caller has already returned is never lost.
9485
9628
  */
9486
- async startWith(workflow, input, runId, opts, parkUnroutable) {
9629
+ async startWith(workflow, input, runId, opts, parkUnroutable, wireSingleton) {
9487
9630
  const name = workflowName(workflow);
9488
9631
  const pinned = opts?.version;
9489
9632
  const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
@@ -9508,12 +9651,15 @@ var WorkflowEngine = class _WorkflowEngine {
9508
9651
  updatedAt: /* @__PURE__ */ new Date()
9509
9652
  };
9510
9653
  const convention = await this.resolveRemoteByConvention(conventionRun, {
9511
- askFleetForVersion: true
9654
+ askFleetForVersion: true,
9655
+ ...wireSingleton !== void 0 ? {
9656
+ singleton: wireSingleton
9657
+ } : {}
9512
9658
  });
9513
9659
  registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
9514
9660
  }
9515
9661
  if (!registered && parkUnroutable) {
9516
- return this.parkStartForWorker(name, input, runId, opts);
9662
+ return this.parkStartForWorker(name, input, runId, opts, wireSingleton);
9517
9663
  }
9518
9664
  if (!registered) {
9519
9665
  throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
@@ -9615,11 +9761,13 @@ var WorkflowEngine = class _WorkflowEngine {
9615
9761
  * declared version replaces), priority, tags, search attributes and customer tenant. A delayed
9616
9762
  * start is checked first at its `startAt`.
9617
9763
  */
9618
- async parkStartForWorker(name, input, runId, opts) {
9764
+ async parkStartForWorker(name, input, runId, opts, wireSingleton) {
9619
9765
  const prior = await this.store.getRun(runId);
9620
9766
  if (prior) return this.existingStartResult(prior);
9621
9767
  const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
9622
9768
  if (quota) await assertConcurrency(this.store, name, quota);
9769
+ const singleton = conventionSingleton({}, void 0, wireSingleton);
9770
+ if (singleton) await this.singletons.assertCapacity(name, singleton, input);
9623
9771
  const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
9624
9772
  const group = tenantGroup(sanitizeQueueToken(name), namespace);
9625
9773
  const pinned = opts?.version;
@@ -9634,6 +9782,9 @@ var WorkflowEngine = class _WorkflowEngine {
9634
9782
  const now = /* @__PURE__ */ new Date();
9635
9783
  const tags = mergeTags(void 0, [
9636
9784
  ...opts?.tags ?? [],
9785
+ ...singleton ? [
9786
+ this.singletons.tag(singleton, input)
9787
+ ] : [],
9637
9788
  ...quota ? [
9638
9789
  concurrencyTag(quota.key)
9639
9790
  ] : [],
@@ -9722,8 +9873,12 @@ var WorkflowEngine = class _WorkflowEngine {
9722
9873
  concurrency: message.concurrency
9723
9874
  } : {}
9724
9875
  };
9876
+ const singleton = message.singleton !== void 0 && typeof message.singleton?.key === "string" ? {
9877
+ ...message.singleton,
9878
+ key: tenantScopedKey(namespace, message.singleton.key)
9879
+ } : void 0;
9725
9880
  try {
9726
- return await this.startWith(message.workflow, message.input, runId, opts, true);
9881
+ return await this.startWith(message.workflow, message.input, runId, opts, true, singleton);
9727
9882
  } catch (err) {
9728
9883
  if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
9729
9884
  throw err;
@@ -9731,6 +9886,62 @@ var WorkflowEngine = class _WorkflowEngine {
9731
9886
  return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
9732
9887
  }
9733
9888
  }
9889
+ /**
9890
+ * The control plane's half of a tenant's `signalWithStart` (a `signalWithStart` run request):
9891
+ * start `start` like {@link startForTenant} unless its run exists, then signal `signal.token`.
9892
+ * Authorization is the caller's (the run-request responder): the run must belong to `namespace`,
9893
+ * and so must a run already waiting on the token. Delivered at once to a waiting run; with none
9894
+ * waiting yet it is buffered only when the token contains the run id (a run-scoped token), so a
9895
+ * tenant can never plant a payload for a wait another namespace opens later. `signalled: false`
9896
+ * when it was neither delivered nor buffered.
9897
+ */
9898
+ async signalWithStartForTenant(start, namespace, tenant, signal) {
9899
+ const started = await this.startForTenant(start, namespace, tenant);
9900
+ if (TERMINAL_RUN_STATUSES.includes(started.status)) {
9901
+ return {
9902
+ runId: start.runId,
9903
+ signalled: false
9904
+ };
9905
+ }
9906
+ const waiter = (await this.store.listSignalWaiters(signal.token)).find((w) => w.token === signal.token);
9907
+ if (waiter) {
9908
+ const run = await this.store.getRun(waiter.runId);
9909
+ if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) {
9910
+ return {
9911
+ runId: start.runId,
9912
+ signalled: false
9913
+ };
9914
+ }
9915
+ } else if (!signal.token.includes(start.runId)) {
9916
+ return {
9917
+ runId: start.runId,
9918
+ signalled: false
9919
+ };
9920
+ }
9921
+ await this.signalUnchecked(signal.token, signal.payload);
9922
+ return {
9923
+ runId: start.runId,
9924
+ signalled: true
9925
+ };
9926
+ }
9927
+ /**
9928
+ * The control plane's half of a tenant's `publishEvent` (a `publishEvent` run request): resume the
9929
+ * runs of `namespace` waiting on event `name` whose match the payload satisfies. Starting the
9930
+ * tenant's own subscribers is the tenant's (it holds their `where` and `input`); nothing is
9931
+ * buffered, since a buffered event could reach another namespace's later wait. Returns how many
9932
+ * runs it resumed.
9933
+ */
9934
+ async publishEventForTenant(name, payload, namespace) {
9935
+ let touched = 0;
9936
+ for (const w of await this.store.listSignalWaiters(eventPrefix(name))) {
9937
+ if (!eventMatches(payload, eventMatchOf(w.token))) continue;
9938
+ const run = await this.store.getRun(w.runId);
9939
+ if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) continue;
9940
+ await this.signalUnchecked(w.token, payload);
9941
+ touched += 1;
9942
+ }
9943
+ return touched;
9944
+ }
9734
9945
  /** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
9735
9946
  async recordRejectedStart(name, input, runId, opts, err) {
9736
9947
  const prior = await this.store.getRun(runId);
@@ -10202,9 +10413,16 @@ var WorkflowEngine = class _WorkflowEngine {
10202
10413
  if (!liveTransport) return void 0;
10203
10414
  const executor = new RemoteWorkflowExecutor(liveTransport, run.workflow, run.namespace);
10204
10415
  const declared = opts.askFleetForVersion ? await this.declaredVersion(liveTransport, group, run.workflow) : void 0;
10416
+ const version = declared ?? run.workflowVersion;
10417
+ const announced = await this.announcedRegistration(group, run.workflow, version);
10418
+ const singleton = conventionSingleton(run, announced, opts.singleton);
10419
+ if (singleton?.limit !== void 0) {
10420
+ this.conventionSingletonLimits.set(`${runNamespace(run.namespace)}\0${run.workflow}`, singleton.limit);
10421
+ }
10422
+ const tags = announced?.options?.tags;
10205
10423
  return {
10206
10424
  name: run.workflow,
10207
- version: declared ?? run.workflowVersion,
10425
+ version,
10208
10426
  fn: /* @__PURE__ */ __name(() => {
10209
10427
  throw new Error(`remote workflow ${run.workflow} has no local body`);
10210
10428
  }, "fn"),
@@ -10213,12 +10431,47 @@ var WorkflowEngine = class _WorkflowEngine {
10213
10431
  group,
10214
10432
  executor
10215
10433
  },
10434
+ ...Array.isArray(tags) && tags.length > 0 ? {
10435
+ tags
10436
+ } : {},
10437
+ ...singleton ? {
10438
+ singleton
10439
+ } : {},
10216
10440
  ...opts.askFleetForVersion ? {
10217
10441
  versionDeclared: declared !== void 0
10218
10442
  } : {}
10219
10443
  };
10220
10444
  }
10221
10445
  /**
10446
+ * The registration the live workers of `group` announce for `workflow` (at `version`, when one of
10447
+ * them states it): where they disagree (a rolling deploy), the one from the worker that started
10448
+ * last, as for announced schedules. `undefined` when none announces it, `null` when the read
10449
+ * failed (what the workers state is then unknown, not "nothing"). Reads
10450
+ * through the descriptor cache ({@link workerDescriptorCacheMs}), so a turn costs no extra round
10451
+ * trip in steady state.
10452
+ */
10453
+ async announcedRegistration(group, workflow, version) {
10454
+ let descriptors;
10455
+ try {
10456
+ descriptors = await this.liveWorkerDescriptors(group);
10457
+ } catch {
10458
+ return null;
10459
+ }
10460
+ let best;
10461
+ for (const descriptor of descriptors) {
10462
+ for (const reg of descriptor.registrations ?? []) {
10463
+ if (reg?.name !== workflow) continue;
10464
+ if (version !== void 0 && reg.version !== void 0 && reg.version !== version) continue;
10465
+ const at = Number(descriptor.startedAt) || 0;
10466
+ if (!best || at > best.at) best = {
10467
+ at,
10468
+ reg
10469
+ };
10470
+ }
10471
+ }
10472
+ return best?.reg;
10473
+ }
10474
+ /**
10222
10475
  * The version the LIVE fleet declares for `workflow` on `group`, or `undefined` when it declares
10223
10476
  * none — which is a fact about the fleet, not a failure to read it.
10224
10477
  *
@@ -10442,34 +10695,95 @@ var WorkflowEngine = class _WorkflowEngine {
10442
10695
  for (const reg of new Set(this.latest.values())) {
10443
10696
  if (this.draining) return;
10444
10697
  if (reg.executionTimeoutMs == null) continue;
10445
- const deadline = now - reg.executionTimeoutMs;
10446
- const inflight = await this.store.listRuns({
10447
- workflow: reg.name,
10448
- statuses: TIMEOUT_SWEPT_STATUSES,
10449
- namespace: this.namespace
10450
- });
10451
- for (const run of inflight) {
10452
- if (this.draining) return;
10453
- if (run.createdAt.getTime() > deadline) continue;
10454
- const error2 = {
10455
- message: "execution timeout",
10456
- code: "execution_timeout"
10457
- };
10458
- await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
10459
- status: "cancelled",
10460
- error: error2,
10461
- updatedAt: /* @__PURE__ */ new Date()
10462
- }));
10463
- this.emit({
10464
- type: "run.failed",
10465
- runId: run.id,
10466
- workflow: run.workflow,
10467
- namespace: run.namespace,
10468
- error: error2
10469
- });
10470
- await this.cancelChildren(run.id).catch(() => void 0);
10471
- this.notifyParent(run.id, cancelledCompletion(error2));
10698
+ await this.cancelTimedOut(reg.name, this.namespace, now - reg.executionTimeoutMs);
10699
+ }
10700
+ for (const { workflow, namespace, timeoutMs } of await this.announcedTimeouts(now)) {
10701
+ if (this.draining) return;
10702
+ await this.cancelTimedOut(workflow, namespace, now - timeoutMs);
10703
+ }
10704
+ }
10705
+ /**
10706
+ * The singleton limit last resolved for a workflow served by convention, by `<namespace>\0<name>`
10707
+ * (see {@link resolveRemoteByConvention}): what the singleton gate's release wake-up reads, since
10708
+ * it cannot wait on the fleet. One entry per convention singleton the fleet serves.
10709
+ */
10710
+ conventionSingletonLimits = /* @__PURE__ */ new Map();
10711
+ /** The last read of the announced execution timeouts (see {@link announcedTimeouts}). */
10712
+ announcedTimeoutsRead;
10713
+ /**
10714
+ * The `executionTimeoutMs` live workers announce (`@Workflow({ executionTimeout })` on a tenant),
10715
+ * per workflow and namespace, for workflows this engine does not register (a registration here
10716
+ * applies its own). Where workers disagree, the one that started last wins. Read at most once per
10717
+ * {@link ANNOUNCED_TIMEOUTS_TTL_MS}; empty without a transport that reads the fleet, or when the
10718
+ * read fails.
10719
+ */
10720
+ async announcedTimeouts(now) {
10721
+ if (!this.pool.introspectsFleet) return [];
10722
+ const cached = this.announcedTimeoutsRead;
10723
+ if (cached && now - cached.at < ANNOUNCED_TIMEOUTS_TTL_MS) return cached.list;
10724
+ const newest = /* @__PURE__ */ new Map();
10725
+ try {
10726
+ for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
10727
+ const at = Number(descriptor.startedAt) || 0;
10728
+ for (const reg of descriptor.registrations ?? []) {
10729
+ if (!reg || typeof reg.name !== "string" || this.latest.has(reg.name)) continue;
10730
+ const namespace = servedNamespace(descriptor.namespace, reg.group);
10731
+ if (this.namespace !== void 0 && runNamespace(this.namespace) !== namespace) continue;
10732
+ const key = `${namespace}\0${reg.name}`;
10733
+ const prior = newest.get(key);
10734
+ if (prior && prior.at >= at) continue;
10735
+ newest.set(key, {
10736
+ at,
10737
+ workflow: reg.name,
10738
+ namespace,
10739
+ ms: Number(reg.options?.executionTimeoutMs)
10740
+ });
10741
+ }
10472
10742
  }
10743
+ } catch {
10744
+ return cached?.list ?? [];
10745
+ }
10746
+ const list2 = [
10747
+ ...newest.values()
10748
+ ].filter((e) => Number.isFinite(e.ms) && e.ms > 0).map((e) => ({
10749
+ workflow: e.workflow,
10750
+ namespace: e.namespace,
10751
+ timeoutMs: e.ms
10752
+ }));
10753
+ this.announcedTimeoutsRead = {
10754
+ at: now,
10755
+ list: list2
10756
+ };
10757
+ return list2;
10758
+ }
10759
+ /** Cancel (`execution_timeout`) the in-flight runs of `workflow` in `namespace` created before `deadline`. */
10760
+ async cancelTimedOut(workflow, namespace, deadline) {
10761
+ const inflight = await this.store.listRuns({
10762
+ workflow,
10763
+ statuses: TIMEOUT_SWEPT_STATUSES,
10764
+ namespace
10765
+ });
10766
+ for (const run of inflight) {
10767
+ if (this.draining) return;
10768
+ if (run.createdAt.getTime() > deadline) continue;
10769
+ const error2 = {
10770
+ message: "execution timeout",
10771
+ code: "execution_timeout"
10772
+ };
10773
+ await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
10774
+ status: "cancelled",
10775
+ error: error2,
10776
+ updatedAt: /* @__PURE__ */ new Date()
10777
+ }));
10778
+ this.emit({
10779
+ type: "run.failed",
10780
+ runId: run.id,
10781
+ workflow: run.workflow,
10782
+ namespace: run.namespace,
10783
+ error: error2
10784
+ });
10785
+ await this.cancelChildren(run.id).catch(() => void 0);
10786
+ this.notifyParent(run.id, cancelledCompletion(error2));
10473
10787
  }
10474
10788
  }
10475
10789
  /**
@@ -13181,7 +13495,7 @@ ${run.workflowVersion}`;
13181
13495
  let wfPlan = await this.planRoute(wfToken, registered.requires);
13182
13496
  if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
13183
13497
  if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
13184
- const checkpoints = await this.replayCheckpoints(run.id);
13498
+ const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
13185
13499
  await this.deliverFinishedChildren(run.id, checkpoints);
13186
13500
  await this.deliverFinishedChildWaiters(run.id, checkpoints);
13187
13501
  const handoff = liveHandoff(run.id, checkpoints);
@@ -13267,6 +13581,10 @@ ${run.workflowVersion}`;
13267
13581
  */
13268
13582
  async applyDecision(run, decision) {
13269
13583
  if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
13584
+ const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
13585
+ if (handoff !== void 0 && decision.status === "completed") {
13586
+ return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
13587
+ }
13270
13588
  if (decision.status === "completed") {
13271
13589
  await this.applyCommands(run, decision.commands);
13272
13590
  return this.settleRun(run, {
@@ -13313,6 +13631,63 @@ ${run.workflowVersion}`;
13313
13631
  return suspended;
13314
13632
  }
13315
13633
  /**
13634
+ * A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
13635
+ * run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
13636
+ * engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
13637
+ * turn replayed stays cancelled and starts nothing.
13638
+ */
13639
+ async applyRemoteContinueAsNew(run, commands, input) {
13640
+ await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
13641
+ const fresh = await this.store.getRun(run.id);
13642
+ if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
13643
+ return {
13644
+ runId: run.id,
13645
+ status: fresh.status,
13646
+ output: fresh.output,
13647
+ error: fresh.error
13648
+ };
13649
+ }
13650
+ const nextId = nextContinuationId(run.id);
13651
+ try {
13652
+ await this.start(run.workflow, input, nextId, {
13653
+ namespace: run.namespace,
13654
+ ...inheritTenant(run)
13655
+ });
13656
+ } catch (err) {
13657
+ const message = err instanceof Error ? err.message : String(err);
13658
+ return this.settleRun(run, {
13659
+ kind: "failed",
13660
+ error: {
13661
+ message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
13662
+ code: "continue_as_new_failed"
13663
+ }
13664
+ });
13665
+ }
13666
+ await this.store.updateRun(run.id, {
13667
+ status: "completed",
13668
+ output: void 0,
13669
+ error: void 0,
13670
+ wakeAt: void 0,
13671
+ updatedAt: /* @__PURE__ */ new Date()
13672
+ });
13673
+ this.emit({
13674
+ type: "run.completed",
13675
+ runId: run.id,
13676
+ workflow: run.workflow,
13677
+ namespace: run.namespace
13678
+ }, {
13679
+ lifecycle: false
13680
+ });
13681
+ void this.notifyParent(run.id, {
13682
+ ok: true,
13683
+ value: void 0
13684
+ });
13685
+ return {
13686
+ runId: run.id,
13687
+ status: "completed"
13688
+ };
13689
+ }
13690
+ /**
13316
13691
  * Re-drive a run whose turn was computed from a history OLDER than the store's.
13317
13692
  *
13318
13693
  * A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
@@ -13330,11 +13705,12 @@ ${run.workflowVersion}`;
13330
13705
  * Scheduled off the current stack so it runs after the caller released the run lease.
13331
13706
  */
13332
13707
  async redriveIfTurnSawStaleHistory(runId, commands) {
13333
- const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
13708
+ const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
13334
13709
  if (blocking.length === 0) return;
13335
13710
  for (const cmd of blocking) {
13336
13711
  const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
13337
- if (checkpoint && checkpoint.status !== "pending") {
13712
+ const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
13713
+ if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
13338
13714
  setTimeout(() => {
13339
13715
  void this.resumeImplicit(runId).catch(() => void 0);
13340
13716
  }, 0).unref?.();
@@ -13583,7 +13959,12 @@ ${run.workflowVersion}`;
13583
13959
  // name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
13584
13960
  name: cp.kind === "signal" ? void 0 : cp.name,
13585
13961
  output: cp.status === "completed" ? cp.output : void 0,
13586
- error: cp.status === "failed" ? cp.error : void 0
13962
+ error: cp.status === "failed" ? cp.error : void 0,
13963
+ // How many times a dispatched step ran: a worker applying a `call` policy decides a retry
13964
+ // from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
13965
+ ...cp.kind === "remote" ? {
13966
+ attempts: cp.attempts
13967
+ } : {}
13587
13968
  });
13588
13969
  } else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
13589
13970
  events.push({
@@ -13640,11 +14021,12 @@ ${run.workflowVersion}`;
13640
14021
  } else if (cmd.kind === "call") {
13641
14022
  const existing = await this.store.getCheckpoint(run.id, cmd.seq);
13642
14023
  if (existing) {
13643
- const leaseAt = redriveLost ? await this.redriveLostCall(run, existing) : void 0;
14024
+ const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
14025
+ const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
13644
14026
  if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
13645
14027
  continue;
13646
14028
  }
13647
- const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14029
+ const lease = this.callLease(cmd.policy);
13648
14030
  await this.store.saveCheckpoint(stepCheckpoint({
13649
14031
  runId: run.id,
13650
14032
  seq: cmd.seq,
@@ -13704,35 +14086,7 @@ ${run.workflowVersion}`;
13704
14086
  }));
13705
14087
  wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
13706
14088
  } else if (cmd.kind === "waitSignal") {
13707
- const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
13708
- await this.store.saveCheckpoint(instantCheckpoint({
13709
- runId: run.id,
13710
- seq: cmd.seq,
13711
- name: `signal:${cmd.signal}`,
13712
- kind: "signal",
13713
- output: payload
13714
- }));
13715
- setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
13716
- }, "deliverBuffered");
13717
- const buffered = await this.store.takeBufferedSignal(cmd.signal);
13718
- if (buffered) {
13719
- await deliverBuffered(buffered.payload);
13720
- } else {
13721
- await this.store.putSignalWaiter({
13722
- token: cmd.signal,
13723
- runId: run.id,
13724
- seq: cmd.seq
13725
- });
13726
- const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
13727
- if (lateBuffered) {
13728
- await this.store.removeSignalWaiter({
13729
- token: cmd.signal,
13730
- runId: run.id,
13731
- seq: cmd.seq
13732
- });
13733
- await deliverBuffered(lateBuffered.payload);
13734
- }
13735
- }
14089
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
13736
14090
  } else if (cmd.kind === "waitAny") {
13737
14091
  const state = await enterWaitAny({
13738
14092
  store: this.store,
@@ -13764,6 +14118,100 @@ ${run.workflowVersion}`;
13764
14118
  } else {
13765
14119
  this.redeliverFinishedChild(childId, existingChild);
13766
14120
  }
14121
+ } else if (cmd.kind === "sleepUntil") {
14122
+ const existing = await this.store.getCheckpoint(run.id, cmd.seq);
14123
+ const deadline = existing?.wakeAt ?? cmd.until;
14124
+ if (!existing) {
14125
+ await this.store.saveCheckpoint(stepCheckpoint({
14126
+ runId: run.id,
14127
+ seq: cmd.seq,
14128
+ name: `sleep:${cmd.seq}`,
14129
+ kind: "sleep",
14130
+ status: "pending",
14131
+ attempts: 1,
14132
+ wakeAt: deadline,
14133
+ enqueuedAt: at,
14134
+ startedAt: at,
14135
+ finishedAt: at
14136
+ }));
14137
+ }
14138
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14139
+ } else if (cmd.kind === "waitSignalTimeout") {
14140
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14141
+ const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
14142
+ waitSeq: cmd.seq,
14143
+ token: cmd.signal
14144
+ });
14145
+ if (deadline <= this.clock()) {
14146
+ this.resumeSoon(run.id);
14147
+ continue;
14148
+ }
14149
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
14150
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14151
+ } else if (cmd.kind === "waitEvent") {
14152
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14153
+ const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
14154
+ let deadline;
14155
+ if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
14156
+ deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
14157
+ waitSeq: cmd.seq,
14158
+ token
14159
+ });
14160
+ if (deadline <= this.clock()) {
14161
+ this.resumeSoon(run.id);
14162
+ continue;
14163
+ }
14164
+ }
14165
+ await this.store.putSignalWaiter({
14166
+ token,
14167
+ runId: run.id,
14168
+ seq: cmd.seq
14169
+ });
14170
+ const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
14171
+ if (hit) {
14172
+ await this.store.removeSignalWaiter({
14173
+ token,
14174
+ runId: run.id,
14175
+ seq: cmd.seq
14176
+ });
14177
+ await this.store.saveCheckpoint(instantCheckpoint({
14178
+ runId: run.id,
14179
+ seq: cmd.seq,
14180
+ name: `signal:${token}`,
14181
+ kind: "signal",
14182
+ output: hit.payload
14183
+ }));
14184
+ this.resumeSoon(run.id);
14185
+ } else if (deadline != null) {
14186
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14187
+ }
14188
+ } else if (cmd.kind === "spawnChild") {
14189
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14190
+ if (!await this.store.getRun(cmd.childId)) {
14191
+ try {
14192
+ await this.start(cmd.workflow, cmd.input, cmd.childId, {
14193
+ namespace: run.namespace,
14194
+ ...inheritTenant(run),
14195
+ ...cmd.priority !== void 0 ? {
14196
+ priority: cmd.priority
14197
+ } : {}
14198
+ });
14199
+ } catch (err) {
14200
+ const message = err instanceof Error ? err.message : String(err);
14201
+ this.notifyParent(cmd.childId, {
14202
+ ok: false,
14203
+ error: `child workflow "${cmd.workflow}" failed to start: ${message}`
14204
+ });
14205
+ }
14206
+ }
14207
+ await this.store.saveCheckpoint(instantCheckpoint({
14208
+ runId: run.id,
14209
+ seq: cmd.seq,
14210
+ name: `spawn:${cmd.childId}`,
14211
+ kind: "local",
14212
+ output: cmd.childId
14213
+ }));
14214
+ } else if (cmd.kind === "continueAsNew") {
13767
14215
  } else {
13768
14216
  throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
13769
14217
  }
@@ -13771,6 +14219,245 @@ ${run.workflowVersion}`;
13771
14219
  return wakeAt;
13772
14220
  }
13773
14221
  /**
14222
+ * Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
14223
+ * the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
14224
+ * (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
14225
+ * re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
14226
+ * bails).
14227
+ */
14228
+ async parkRemoteOnSignal(run, seq, token) {
14229
+ const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
14230
+ await this.store.saveCheckpoint(instantCheckpoint({
14231
+ runId: run.id,
14232
+ seq,
14233
+ name: `signal:${token}`,
14234
+ kind: "signal",
14235
+ output: payload
14236
+ }));
14237
+ this.resumeSoon(run.id);
14238
+ }, "deliverBuffered");
14239
+ const buffered = await this.store.takeBufferedSignal(token);
14240
+ if (buffered) {
14241
+ await deliverBuffered(buffered.payload);
14242
+ } else {
14243
+ await this.store.putSignalWaiter({
14244
+ token,
14245
+ runId: run.id,
14246
+ seq
14247
+ });
14248
+ const lateBuffered = await this.store.takeBufferedSignal(token);
14249
+ if (lateBuffered) {
14250
+ await this.store.removeSignalWaiter({
14251
+ token,
14252
+ runId: run.id,
14253
+ seq
14254
+ });
14255
+ await deliverBuffered(lateBuffered.payload);
14256
+ }
14257
+ }
14258
+ }
14259
+ /** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
14260
+ resumeSoon(runId) {
14261
+ setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
14262
+ }
14263
+ /**
14264
+ * Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
14265
+ * `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
14266
+ * back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
14267
+ * deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
14268
+ */
14269
+ async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
14270
+ const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
14271
+ if (recorded?.wakeAt != null) return recorded.wakeAt;
14272
+ const deadline = this.clock() + timeoutMs;
14273
+ const at = /* @__PURE__ */ new Date();
14274
+ await this.store.saveCheckpoint(stepCheckpoint({
14275
+ runId,
14276
+ seq: deadlineSeq,
14277
+ name: `timeout:${label}`,
14278
+ kind: "sleep",
14279
+ status: "pending",
14280
+ attempts: 1,
14281
+ input: wait,
14282
+ wakeAt: deadline,
14283
+ enqueuedAt: at,
14284
+ startedAt: at,
14285
+ finishedAt: at
14286
+ }));
14287
+ return deadline;
14288
+ }
14289
+ /**
14290
+ * Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
14291
+ * (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
14292
+ * time out — the in-process wait removes its waiter the same way when it finds the deadline past.
14293
+ * A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
14294
+ * worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
14295
+ */
14296
+ async expireRemoteDeadlines(runId, checkpoints) {
14297
+ const now = this.clock();
14298
+ const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
14299
+ let out = checkpoints;
14300
+ for (const cp of checkpoints) {
14301
+ if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
14302
+ if (cp.wakeAt == null || cp.wakeAt > now) continue;
14303
+ const wait = cp.input;
14304
+ if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
14305
+ if (settled.has(wait.waitSeq)) continue;
14306
+ await this.store.removeSignalWaiter({
14307
+ token: wait.token,
14308
+ runId,
14309
+ seq: wait.waitSeq
14310
+ });
14311
+ const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
14312
+ if (landed && landed.status === "completed") {
14313
+ out = [
14314
+ ...out.filter((c) => c.seq !== landed.seq),
14315
+ landed
14316
+ ].sort((a, b) => a.seq - b.seq);
14317
+ }
14318
+ }
14319
+ return out;
14320
+ }
14321
+ /**
14322
+ * Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
14323
+ * see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
14324
+ * candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
14325
+ * {@link eventBufferTtlMs} is set, like the reconcile sweep.
14326
+ */
14327
+ async claimBufferedEvent(run, name, match) {
14328
+ const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
14329
+ const nowMs = this.clock();
14330
+ for (const candidate of candidates) {
14331
+ if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
14332
+ await this.store.removeBufferedEvent(candidate.id);
14333
+ continue;
14334
+ }
14335
+ const { payload, tenantIds } = readBufferedEvent(candidate.payload);
14336
+ if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
14337
+ if (!eventMatches(payload, match)) continue;
14338
+ if (!await this.store.removeBufferedEvent(candidate.id)) continue;
14339
+ return {
14340
+ payload
14341
+ };
14342
+ }
14343
+ return void 0;
14344
+ }
14345
+ /** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
14346
+ * sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
14347
+ callLease(policy) {
14348
+ if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
14349
+ return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14350
+ }
14351
+ /**
14352
+ * A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
14353
+ * checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
14354
+ * callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
14355
+ * behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
14356
+ * timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
14357
+ * and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
14358
+ *
14359
+ * - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
14360
+ * the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
14361
+ * dispatched again under the same `stepId` with `attempts + 1`.
14362
+ * - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
14363
+ * re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
14364
+ * workflow can catch, as `RemoteStepTimeout` would surface in-process.
14365
+ */
14366
+ async applyCallPolicy(run, cmd, cp) {
14367
+ const policy = cmd.policy;
14368
+ if (cp.kind !== "remote") return null;
14369
+ if (cp.status === "failed") {
14370
+ if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
14371
+ let due = cp.wakeAt;
14372
+ if (due == null) {
14373
+ due = this.clock() + backoffDelay(cp.attempts, policy);
14374
+ if (due > this.clock()) {
14375
+ await this.store.saveCheckpoint({
14376
+ ...cp,
14377
+ wakeAt: due
14378
+ });
14379
+ return due;
14380
+ }
14381
+ }
14382
+ if (this.clock() < due) return due;
14383
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
14384
+ }
14385
+ if (cp.status !== "pending" || !policy.timeoutMs) return null;
14386
+ if (cp.wakeAt == null) {
14387
+ const lease = this.clock() + policy.timeoutMs;
14388
+ await this.store.saveCheckpoint({
14389
+ ...cp,
14390
+ wakeAt: lease
14391
+ });
14392
+ return this.stepLeaseWakeAt(lease);
14393
+ }
14394
+ if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
14395
+ if (cp.attempts < remoteCallMaxAttempts(policy)) {
14396
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
14397
+ }
14398
+ const id = cp.stepId ?? stepId(run.id, cp.seq);
14399
+ const error2 = {
14400
+ message: new RemoteStepTimeout(id, policy.timeoutMs).message,
14401
+ code: "remote_step_timeout"
14402
+ };
14403
+ await this.store.saveCheckpoint({
14404
+ ...cp,
14405
+ status: "failed",
14406
+ error: error2,
14407
+ wakeAt: void 0,
14408
+ finishedAt: /* @__PURE__ */ new Date()
14409
+ });
14410
+ this.emit({
14411
+ type: "step.failed",
14412
+ runId: run.id,
14413
+ seq: cp.seq,
14414
+ name: cp.name,
14415
+ kind: "remote",
14416
+ error: error2
14417
+ });
14418
+ return void 0;
14419
+ }
14420
+ /** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
14421
+ async redispatchCall(run, cmd, cp, attempt, policy, why) {
14422
+ const lease = this.callLease(policy);
14423
+ const at = /* @__PURE__ */ new Date();
14424
+ await this.store.saveCheckpoint({
14425
+ ...cp,
14426
+ status: "pending",
14427
+ output: void 0,
14428
+ error: void 0,
14429
+ attempts: attempt,
14430
+ wakeAt: lease,
14431
+ enqueuedAt: at,
14432
+ startedAt: at,
14433
+ finishedAt: at,
14434
+ events: why === "timeout" ? [
14435
+ ...cp.events ?? [],
14436
+ lostStepEvent(this.clock(), attempt, "redispatched")
14437
+ ] : cp.events
14438
+ });
14439
+ await this.dispatchRemoteTask({
14440
+ runId: run.id,
14441
+ seq: cp.seq,
14442
+ name: cmd.name,
14443
+ stepId: cp.stepId ?? stepId(run.id, cp.seq),
14444
+ group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
14445
+ input: cp.input,
14446
+ attempt
14447
+ });
14448
+ this.emit({
14449
+ type: "step.started",
14450
+ runId: run.id,
14451
+ seq: cp.seq,
14452
+ name: cmd.name,
14453
+ kind: "remote",
14454
+ ...why === "timeout" ? {
14455
+ redispatched: true
14456
+ } : {}
14457
+ });
14458
+ return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
14459
+ }
14460
+ /**
13774
14461
  * Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
13775
14462
  * exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
13776
14463
  * suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
@@ -18171,6 +18858,7 @@ export {
18171
18858
  PARENT_CLOSE_TAG_PREFIX,
18172
18859
  QueueController,
18173
18860
  REMOTE_EVENT,
18861
+ REMOTE_WORKFLOW_FEATURES,
18174
18862
  RUN_HISTORY_FIXTURE_VERSION,
18175
18863
  RUN_VALUE_FACET_LIMIT,
18176
18864
  RUN_VALUE_FACET_SCAN,
@@ -18328,6 +19016,9 @@ export {
18328
19016
  readBufferedEvent,
18329
19017
  readRunsByIds,
18330
19018
  readSearchAttributes,
19019
+ remoteCallMaxAttempts,
19020
+ remoteCallPolicyOf,
19021
+ remoteCallRetries,
18331
19022
  requiredCapabilities,
18332
19023
  resolveConcurrency,
18333
19024
  resolveRouting,