@dudousxd/nestjs-durable-core 0.101.0 → 0.103.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -77,6 +77,7 @@ __export(index_exports, {
77
77
  PARENT_CLOSE_TAG_PREFIX: () => PARENT_CLOSE_TAG_PREFIX,
78
78
  QueueController: () => QueueController,
79
79
  REMOTE_EVENT: () => REMOTE_EVENT,
80
+ REMOTE_WORKFLOW_FEATURES: () => REMOTE_WORKFLOW_FEATURES,
80
81
  RUN_HISTORY_FIXTURE_VERSION: () => RUN_HISTORY_FIXTURE_VERSION,
81
82
  RUN_VALUE_FACET_LIMIT: () => RUN_VALUE_FACET_LIMIT,
82
83
  RUN_VALUE_FACET_SCAN: () => RUN_VALUE_FACET_SCAN,
@@ -234,6 +235,9 @@ __export(index_exports, {
234
235
  readBufferedEvent: () => readBufferedEvent,
235
236
  readRunsByIds: () => readRunsByIds,
236
237
  readSearchAttributes: () => readSearchAttributes,
238
+ remoteCallMaxAttempts: () => remoteCallMaxAttempts,
239
+ remoteCallPolicyOf: () => remoteCallPolicyOf,
240
+ remoteCallRetries: () => remoteCallRetries,
237
241
  requiredCapabilities: () => requiredCapabilities,
238
242
  resolveConcurrency: () => resolveConcurrency,
239
243
  resolveRouting: () => resolveRouting,
@@ -1178,6 +1182,14 @@ function isNonEmptyRunScope(scope) {
1178
1182
  return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
1179
1183
  }
1180
1184
  __name(isNonEmptyRunScope, "isNonEmptyRunScope");
1185
+ var REMOTE_WORKFLOW_FEATURES = [
1186
+ "callPolicy",
1187
+ "sleepUntil",
1188
+ "waitSignalTimeout",
1189
+ "waitEvent",
1190
+ "spawnChild",
1191
+ "continueAsNew"
1192
+ ];
1181
1193
  var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
1182
1194
 
1183
1195
  // src/events.ts
@@ -2639,6 +2651,13 @@ function canonicalizeRegistrations(regs) {
2639
2651
  subscribes: [
2640
2652
  ...new Set(r.subscribes)
2641
2653
  ].sort()
2654
+ } : {},
2655
+ // Conditional too: a registration without them keeps its hash; a changed option changes it.
2656
+ ...r.options !== void 0 ? {
2657
+ options: stableJson(r.options)
2658
+ } : {},
2659
+ ...r.deadLetter !== void 0 ? {
2660
+ deadLetter: r.deadLetter
2642
2661
  } : {}
2643
2662
  }));
2644
2663
  const byJson = /* @__PURE__ */ new Map();
@@ -3359,6 +3378,35 @@ function codeAndDetails(err) {
3359
3378
  }
3360
3379
  __name(codeAndDetails, "codeAndDetails");
3361
3380
 
3381
+ // src/remote-call-policy.ts
3382
+ function remoteCallPolicyOf(config, opts) {
3383
+ const policy = {};
3384
+ const retries = opts?.retries ?? config?.retries;
3385
+ if (retries !== void 0) policy.retries = retries;
3386
+ const backoff = opts?.backoff ?? config?.backoff;
3387
+ if (backoff !== void 0) policy.backoff = backoff;
3388
+ const backoffMs = opts?.backoffMs ?? config?.backoffMs;
3389
+ if (backoffMs !== void 0) policy.backoffMs = backoffMs;
3390
+ const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
3391
+ if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
3392
+ const jitter = opts?.jitter ?? config?.jitter;
3393
+ if (jitter !== void 0) policy.jitter = jitter;
3394
+ const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
3395
+ if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
3396
+ return Object.keys(policy).length > 0 ? policy : void 0;
3397
+ }
3398
+ __name(remoteCallPolicyOf, "remoteCallPolicyOf");
3399
+ function remoteCallMaxAttempts(policy) {
3400
+ return Math.max(1, policy?.retries ?? 1);
3401
+ }
3402
+ __name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
3403
+ function remoteCallRetries(policy, attempts, error2) {
3404
+ if (policy === void 0 || policy.timeoutMs) return false;
3405
+ if (error2?.retryable === false) return false;
3406
+ return attempts < remoteCallMaxAttempts(policy);
3407
+ }
3408
+ __name(remoteCallRetries, "remoteCallRetries");
3409
+
3362
3410
  // src/remote-event.ts
3363
3411
  var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
3364
3412
  function markRemoteEvent(event) {
@@ -3416,7 +3464,12 @@ var RemoteWorkflowExecutor = class {
3416
3464
  } : {},
3417
3465
  group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
3418
3466
  priority: run.priority,
3419
- attempt: 1
3467
+ attempt: 1,
3468
+ // What this control plane applies beyond the original command kinds, so a worker fails a turn
3469
+ // that needs more instead of emitting a command this engine would not understand.
3470
+ features: [
3471
+ ...REMOTE_WORKFLOW_FEATURES
3472
+ ]
3420
3473
  };
3421
3474
  await this.transport.dispatchWorkflowTask(task);
3422
3475
  }
@@ -4349,7 +4402,8 @@ var ScheduleClient = class {
4349
4402
  const input = specOf(record).input;
4350
4403
  const opts = startOptions(record);
4351
4404
  if (isAnnounced(record) && this.host.startParked) {
4352
- return this.host.startParked(record.workflow, input, runId, opts);
4405
+ const singleton = record.state.announce?.singleton;
4406
+ return singleton ? this.host.startParked(record.workflow, input, runId, opts, singleton) : this.host.startParked(record.workflow, input, runId, opts);
4353
4407
  }
4354
4408
  return this.host.start(record.workflow, input, runId, opts);
4355
4409
  }
@@ -4545,7 +4599,8 @@ var ScheduleClient = class {
4545
4599
  announce: {
4546
4600
  key: doc.key,
4547
4601
  fingerprint,
4548
- declaredPaused: doc.declaredPaused
4602
+ declaredPaused: doc.declaredPaused,
4603
+ singleton: doc.singleton
4549
4604
  }
4550
4605
  }),
4551
4606
  updatedAt: new Date(now)
@@ -4582,7 +4637,10 @@ var ScheduleClient = class {
4582
4637
  announce: {
4583
4638
  key: item.key,
4584
4639
  fingerprint,
4585
- declaredPaused
4640
+ declaredPaused,
4641
+ ...validSingleton(item.singleton) ? {
4642
+ singleton: item.singleton
4643
+ } : {}
4586
4644
  }
4587
4645
  });
4588
4646
  out.adopted.push(id);
@@ -4732,6 +4790,10 @@ function specOf(record) {
4732
4790
  return record.spec;
4733
4791
  }
4734
4792
  __name(specOf, "specOf");
4793
+ function validSingleton(value) {
4794
+ return value !== null && typeof value === "object" && typeof value.key === "string" && value.key !== "";
4795
+ }
4796
+ __name(validSingleton, "validSingleton");
4735
4797
  function isAnnounced(record) {
4736
4798
  return record.state?.announce !== void 0;
4737
4799
  }
@@ -7175,6 +7237,43 @@ var DEFAULT_STICKY_RUNS = 1e3;
7175
7237
  var DEFAULT_STICKY_IDLE_MS = 3e5;
7176
7238
  var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
7177
7239
  var ANNOUNCED_ROUTES_TTL_MS = 5e3;
7240
+ var ANNOUNCED_TIMEOUTS_TTL_MS = 15e3;
7241
+ function tenantScopedKey(namespace, key) {
7242
+ return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
7243
+ }
7244
+ __name(tenantScopedKey, "tenantScopedKey");
7245
+ function singletonKeyOf(run) {
7246
+ const tag = run.tags?.find((t) => t.startsWith("singleton:"));
7247
+ return tag?.slice("singleton:".length);
7248
+ }
7249
+ __name(singletonKeyOf, "singletonKeyOf");
7250
+ function conventionSingleton(run, announced, wire) {
7251
+ if (wire !== void 0) {
7252
+ const key2 = wire.key;
7253
+ return {
7254
+ key: /* @__PURE__ */ __name(() => key2, "key"),
7255
+ ...typeof wire.limit === "number" ? {
7256
+ limit: wire.limit
7257
+ } : {},
7258
+ ...typeof wire.maxQueueDepth === "number" ? {
7259
+ maxQueueDepth: wire.maxQueueDepth
7260
+ } : {}
7261
+ };
7262
+ }
7263
+ const key = singletonKeyOf(run);
7264
+ if (key === void 0 || announced === void 0) return void 0;
7265
+ if (announced === null) return {
7266
+ key: /* @__PURE__ */ __name(() => key, "key"),
7267
+ limit: 1
7268
+ };
7269
+ const declared = announced.options?.singleton;
7270
+ if (declared === void 0) return void 0;
7271
+ return {
7272
+ key: /* @__PURE__ */ __name(() => key, "key"),
7273
+ limit: typeof declared.limit === "number" ? declared.limit : 1
7274
+ };
7275
+ }
7276
+ __name(conventionSingleton, "conventionSingleton");
7178
7277
  function servedNamespace(namespace, group) {
7179
7278
  if (namespace) return namespace;
7180
7279
  const at = group?.lastIndexOf("@") ?? -1;
@@ -7664,13 +7763,28 @@ var WorkflowEngine = class _WorkflowEngine {
7664
7763
  if (this.queuesRunsOverTransport) void this.wake(runId);
7665
7764
  else this.runDispatcher.dispatch(runId);
7666
7765
  }, "dispatch"),
7667
- configFor: /* @__PURE__ */ __name((run) => (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton, "configFor")
7766
+ configFor: /* @__PURE__ */ __name((run) => {
7767
+ const local = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton;
7768
+ if (local) return local;
7769
+ const key = this.latest.has(run.workflow) ? void 0 : singletonKeyOf(run);
7770
+ if (key === void 0) return void 0;
7771
+ const limit = this.conventionSingletonLimits.get(`${runNamespace(run.namespace)}\0${run.workflow}`);
7772
+ return {
7773
+ key: /* @__PURE__ */ __name(() => key, "key"),
7774
+ ...limit !== void 0 ? {
7775
+ limit
7776
+ } : {}
7777
+ };
7778
+ }, "configFor")
7668
7779
  });
7669
7780
  this.schedules = new ScheduleClient(this.store, {
7670
7781
  start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
7671
7782
  // An announced (tenant) schedule fires through the tenant start path: with no live worker
7672
7783
  // of its app, the run waits `blocked` instead of the window being lost.
7673
- startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true), "startParked"),
7784
+ startParked: /* @__PURE__ */ __name((workflow, input, runId, opts, singleton) => this.startWith(workflow, input, runId, opts, true, singleton ? {
7785
+ ...singleton,
7786
+ key: tenantScopedKey(opts?.namespace, singleton.key)
7787
+ } : void 0), "startParked"),
7674
7788
  getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
7675
7789
  isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
7676
7790
  tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
@@ -8480,6 +8594,39 @@ var WorkflowEngine = class _WorkflowEngine {
8480
8594
  } catch {
8481
8595
  }
8482
8596
  }
8597
+ if (!this.latest.has(run.workflow)) this.inBackground(this.routeAnnouncedDeadLetter(run));
8598
+ }
8599
+ /**
8600
+ * The dead-letter workflow the live workers serving `run`'s workflow announce (a tenant's inline
8601
+ * `@DeadLetter()` handler, its `deadLetterWorkflow`, or its module-level default), or `undefined`
8602
+ * when the workflow is registered on this engine (its own routing applies), they announce none, or
8603
+ * they cannot be read. A dead-letter listener with a catch-all default (`DurableModule`'s
8604
+ * `deadLetterWorkflow`) checks it first, so the app's own route wins over the default.
8605
+ */
8606
+ async announcedDeadLetter(run) {
8607
+ if (this.latest.has(run.workflow) || this.isRegisteredHere(run)) return void 0;
8608
+ const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
8609
+ const reg = await this.announcedRegistration(group, run.workflow, run.workflowVersion);
8610
+ const target = reg?.deadLetter;
8611
+ return typeof target === "string" && target !== "" ? target : void 0;
8612
+ }
8613
+ /**
8614
+ * Start the announced dead-letter workflow of a dead run (see {@link announcedDeadLetter}) in the
8615
+ * run's namespace, as `dlq:<runId>` with the `DeadLetter` payload, through the tenant start path:
8616
+ * with no live worker of the app it waits `blocked` instead of being lost. Idempotent by run id.
8617
+ */
8618
+ async routeAnnouncedDeadLetter(run) {
8619
+ const target = await this.announcedDeadLetter(run);
8620
+ if (!target) return;
8621
+ await this.startWith(target, {
8622
+ deadRunId: run.id,
8623
+ workflow: run.workflow,
8624
+ input: run.input,
8625
+ error: run.error
8626
+ }, `dlq:${run.id}`, {
8627
+ namespace: run.namespace,
8628
+ ...inheritTenant(run)
8629
+ }, true);
8483
8630
  }
8484
8631
  /**
8485
8632
  * Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
@@ -9780,7 +9927,7 @@ var WorkflowEngine = class _WorkflowEngine {
9780
9927
  * the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
9781
9928
  * a start whose caller has already returned is never lost.
9782
9929
  */
9783
- async startWith(workflow, input, runId, opts, parkUnroutable) {
9930
+ async startWith(workflow, input, runId, opts, parkUnroutable, wireSingleton) {
9784
9931
  const name = workflowName(workflow);
9785
9932
  const pinned = opts?.version;
9786
9933
  const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
@@ -9805,12 +9952,15 @@ var WorkflowEngine = class _WorkflowEngine {
9805
9952
  updatedAt: /* @__PURE__ */ new Date()
9806
9953
  };
9807
9954
  const convention = await this.resolveRemoteByConvention(conventionRun, {
9808
- askFleetForVersion: true
9955
+ askFleetForVersion: true,
9956
+ ...wireSingleton !== void 0 ? {
9957
+ singleton: wireSingleton
9958
+ } : {}
9809
9959
  });
9810
9960
  registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
9811
9961
  }
9812
9962
  if (!registered && parkUnroutable) {
9813
- return this.parkStartForWorker(name, input, runId, opts);
9963
+ return this.parkStartForWorker(name, input, runId, opts, wireSingleton);
9814
9964
  }
9815
9965
  if (!registered) {
9816
9966
  throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
@@ -9912,11 +10062,13 @@ var WorkflowEngine = class _WorkflowEngine {
9912
10062
  * declared version replaces), priority, tags, search attributes and customer tenant. A delayed
9913
10063
  * start is checked first at its `startAt`.
9914
10064
  */
9915
- async parkStartForWorker(name, input, runId, opts) {
10065
+ async parkStartForWorker(name, input, runId, opts, wireSingleton) {
9916
10066
  const prior = await this.store.getRun(runId);
9917
10067
  if (prior) return this.existingStartResult(prior);
9918
10068
  const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
9919
10069
  if (quota) await assertConcurrency(this.store, name, quota);
10070
+ const singleton = conventionSingleton({}, void 0, wireSingleton);
10071
+ if (singleton) await this.singletons.assertCapacity(name, singleton, input);
9920
10072
  const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
9921
10073
  const group = tenantGroup(sanitizeQueueToken(name), namespace);
9922
10074
  const pinned = opts?.version;
@@ -9931,6 +10083,9 @@ var WorkflowEngine = class _WorkflowEngine {
9931
10083
  const now = /* @__PURE__ */ new Date();
9932
10084
  const tags = mergeTags(void 0, [
9933
10085
  ...opts?.tags ?? [],
10086
+ ...singleton ? [
10087
+ this.singletons.tag(singleton, input)
10088
+ ] : [],
9934
10089
  ...quota ? [
9935
10090
  concurrencyTag(quota.key)
9936
10091
  ] : [],
@@ -10019,8 +10174,12 @@ var WorkflowEngine = class _WorkflowEngine {
10019
10174
  concurrency: message.concurrency
10020
10175
  } : {}
10021
10176
  };
10177
+ const singleton = message.singleton !== void 0 && typeof message.singleton?.key === "string" ? {
10178
+ ...message.singleton,
10179
+ key: tenantScopedKey(namespace, message.singleton.key)
10180
+ } : void 0;
10022
10181
  try {
10023
- return await this.startWith(message.workflow, message.input, runId, opts, true);
10182
+ return await this.startWith(message.workflow, message.input, runId, opts, true, singleton);
10024
10183
  } catch (err) {
10025
10184
  if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
10026
10185
  throw err;
@@ -10028,6 +10187,62 @@ var WorkflowEngine = class _WorkflowEngine {
10028
10187
  return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
10029
10188
  }
10030
10189
  }
10190
+ /**
10191
+ * The control plane's half of a tenant's `signalWithStart` (a `signalWithStart` run request):
10192
+ * start `start` like {@link startForTenant} unless its run exists, then signal `signal.token`.
10193
+ * Authorization is the caller's (the run-request responder): the run must belong to `namespace`,
10194
+ * and so must a run already waiting on the token. Delivered at once to a waiting run; with none
10195
+ * waiting yet it is buffered only when the token contains the run id (a run-scoped token), so a
10196
+ * tenant can never plant a payload for a wait another namespace opens later. `signalled: false`
10197
+ * when it was neither delivered nor buffered.
10198
+ */
10199
+ async signalWithStartForTenant(start, namespace, tenant, signal) {
10200
+ const started = await this.startForTenant(start, namespace, tenant);
10201
+ if (TERMINAL_RUN_STATUSES.includes(started.status)) {
10202
+ return {
10203
+ runId: start.runId,
10204
+ signalled: false
10205
+ };
10206
+ }
10207
+ const waiter = (await this.store.listSignalWaiters(signal.token)).find((w) => w.token === signal.token);
10208
+ if (waiter) {
10209
+ const run = await this.store.getRun(waiter.runId);
10210
+ if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) {
10211
+ return {
10212
+ runId: start.runId,
10213
+ signalled: false
10214
+ };
10215
+ }
10216
+ } else if (!signal.token.includes(start.runId)) {
10217
+ return {
10218
+ runId: start.runId,
10219
+ signalled: false
10220
+ };
10221
+ }
10222
+ await this.signalUnchecked(signal.token, signal.payload);
10223
+ return {
10224
+ runId: start.runId,
10225
+ signalled: true
10226
+ };
10227
+ }
10228
+ /**
10229
+ * The control plane's half of a tenant's `publishEvent` (a `publishEvent` run request): resume the
10230
+ * runs of `namespace` waiting on event `name` whose match the payload satisfies. Starting the
10231
+ * tenant's own subscribers is the tenant's (it holds their `where` and `input`); nothing is
10232
+ * buffered, since a buffered event could reach another namespace's later wait. Returns how many
10233
+ * runs it resumed.
10234
+ */
10235
+ async publishEventForTenant(name, payload, namespace) {
10236
+ let touched = 0;
10237
+ for (const w of await this.store.listSignalWaiters(eventPrefix(name))) {
10238
+ if (!eventMatches(payload, eventMatchOf(w.token))) continue;
10239
+ const run = await this.store.getRun(w.runId);
10240
+ if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) continue;
10241
+ await this.signalUnchecked(w.token, payload);
10242
+ touched += 1;
10243
+ }
10244
+ return touched;
10245
+ }
10031
10246
  /** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
10032
10247
  async recordRejectedStart(name, input, runId, opts, err) {
10033
10248
  const prior = await this.store.getRun(runId);
@@ -10499,9 +10714,16 @@ var WorkflowEngine = class _WorkflowEngine {
10499
10714
  if (!liveTransport) return void 0;
10500
10715
  const executor = new RemoteWorkflowExecutor(liveTransport, run.workflow, run.namespace);
10501
10716
  const declared = opts.askFleetForVersion ? await this.declaredVersion(liveTransport, group, run.workflow) : void 0;
10717
+ const version = declared ?? run.workflowVersion;
10718
+ const announced = await this.announcedRegistration(group, run.workflow, version);
10719
+ const singleton = conventionSingleton(run, announced, opts.singleton);
10720
+ if (singleton?.limit !== void 0) {
10721
+ this.conventionSingletonLimits.set(`${runNamespace(run.namespace)}\0${run.workflow}`, singleton.limit);
10722
+ }
10723
+ const tags = announced?.options?.tags;
10502
10724
  return {
10503
10725
  name: run.workflow,
10504
- version: declared ?? run.workflowVersion,
10726
+ version,
10505
10727
  fn: /* @__PURE__ */ __name(() => {
10506
10728
  throw new Error(`remote workflow ${run.workflow} has no local body`);
10507
10729
  }, "fn"),
@@ -10510,12 +10732,47 @@ var WorkflowEngine = class _WorkflowEngine {
10510
10732
  group,
10511
10733
  executor
10512
10734
  },
10735
+ ...Array.isArray(tags) && tags.length > 0 ? {
10736
+ tags
10737
+ } : {},
10738
+ ...singleton ? {
10739
+ singleton
10740
+ } : {},
10513
10741
  ...opts.askFleetForVersion ? {
10514
10742
  versionDeclared: declared !== void 0
10515
10743
  } : {}
10516
10744
  };
10517
10745
  }
10518
10746
  /**
10747
+ * The registration the live workers of `group` announce for `workflow` (at `version`, when one of
10748
+ * them states it): where they disagree (a rolling deploy), the one from the worker that started
10749
+ * last, as for announced schedules. `undefined` when none announces it, `null` when the read
10750
+ * failed (what the workers state is then unknown, not "nothing"). Reads
10751
+ * through the descriptor cache ({@link workerDescriptorCacheMs}), so a turn costs no extra round
10752
+ * trip in steady state.
10753
+ */
10754
+ async announcedRegistration(group, workflow, version) {
10755
+ let descriptors;
10756
+ try {
10757
+ descriptors = await this.liveWorkerDescriptors(group);
10758
+ } catch {
10759
+ return null;
10760
+ }
10761
+ let best;
10762
+ for (const descriptor of descriptors) {
10763
+ for (const reg of descriptor.registrations ?? []) {
10764
+ if (reg?.name !== workflow) continue;
10765
+ if (version !== void 0 && reg.version !== void 0 && reg.version !== version) continue;
10766
+ const at = Number(descriptor.startedAt) || 0;
10767
+ if (!best || at > best.at) best = {
10768
+ at,
10769
+ reg
10770
+ };
10771
+ }
10772
+ }
10773
+ return best?.reg;
10774
+ }
10775
+ /**
10519
10776
  * The version the LIVE fleet declares for `workflow` on `group`, or `undefined` when it declares
10520
10777
  * none — which is a fact about the fleet, not a failure to read it.
10521
10778
  *
@@ -10739,34 +10996,95 @@ var WorkflowEngine = class _WorkflowEngine {
10739
10996
  for (const reg of new Set(this.latest.values())) {
10740
10997
  if (this.draining) return;
10741
10998
  if (reg.executionTimeoutMs == null) continue;
10742
- const deadline = now - reg.executionTimeoutMs;
10743
- const inflight = await this.store.listRuns({
10744
- workflow: reg.name,
10745
- statuses: TIMEOUT_SWEPT_STATUSES,
10746
- namespace: this.namespace
10747
- });
10748
- for (const run of inflight) {
10749
- if (this.draining) return;
10750
- if (run.createdAt.getTime() > deadline) continue;
10751
- const error2 = {
10752
- message: "execution timeout",
10753
- code: "execution_timeout"
10754
- };
10755
- await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
10756
- status: "cancelled",
10757
- error: error2,
10758
- updatedAt: /* @__PURE__ */ new Date()
10759
- }));
10760
- this.emit({
10761
- type: "run.failed",
10762
- runId: run.id,
10763
- workflow: run.workflow,
10764
- namespace: run.namespace,
10765
- error: error2
10766
- });
10767
- await this.cancelChildren(run.id).catch(() => void 0);
10768
- this.notifyParent(run.id, cancelledCompletion(error2));
10999
+ await this.cancelTimedOut(reg.name, this.namespace, now - reg.executionTimeoutMs);
11000
+ }
11001
+ for (const { workflow, namespace, timeoutMs } of await this.announcedTimeouts(now)) {
11002
+ if (this.draining) return;
11003
+ await this.cancelTimedOut(workflow, namespace, now - timeoutMs);
11004
+ }
11005
+ }
11006
+ /**
11007
+ * The singleton limit last resolved for a workflow served by convention, by `<namespace>\0<name>`
11008
+ * (see {@link resolveRemoteByConvention}): what the singleton gate's release wake-up reads, since
11009
+ * it cannot wait on the fleet. One entry per convention singleton the fleet serves.
11010
+ */
11011
+ conventionSingletonLimits = /* @__PURE__ */ new Map();
11012
+ /** The last read of the announced execution timeouts (see {@link announcedTimeouts}). */
11013
+ announcedTimeoutsRead;
11014
+ /**
11015
+ * The `executionTimeoutMs` live workers announce (`@Workflow({ executionTimeout })` on a tenant),
11016
+ * per workflow and namespace, for workflows this engine does not register (a registration here
11017
+ * applies its own). Where workers disagree, the one that started last wins. Read at most once per
11018
+ * {@link ANNOUNCED_TIMEOUTS_TTL_MS}; empty without a transport that reads the fleet, or when the
11019
+ * read fails.
11020
+ */
11021
+ async announcedTimeouts(now) {
11022
+ if (!this.pool.introspectsFleet) return [];
11023
+ const cached = this.announcedTimeoutsRead;
11024
+ if (cached && now - cached.at < ANNOUNCED_TIMEOUTS_TTL_MS) return cached.list;
11025
+ const newest = /* @__PURE__ */ new Map();
11026
+ try {
11027
+ for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
11028
+ const at = Number(descriptor.startedAt) || 0;
11029
+ for (const reg of descriptor.registrations ?? []) {
11030
+ if (!reg || typeof reg.name !== "string" || this.latest.has(reg.name)) continue;
11031
+ const namespace = servedNamespace(descriptor.namespace, reg.group);
11032
+ if (this.namespace !== void 0 && runNamespace(this.namespace) !== namespace) continue;
11033
+ const key = `${namespace}\0${reg.name}`;
11034
+ const prior = newest.get(key);
11035
+ if (prior && prior.at >= at) continue;
11036
+ newest.set(key, {
11037
+ at,
11038
+ workflow: reg.name,
11039
+ namespace,
11040
+ ms: Number(reg.options?.executionTimeoutMs)
11041
+ });
11042
+ }
10769
11043
  }
11044
+ } catch {
11045
+ return cached?.list ?? [];
11046
+ }
11047
+ const list2 = [
11048
+ ...newest.values()
11049
+ ].filter((e) => Number.isFinite(e.ms) && e.ms > 0).map((e) => ({
11050
+ workflow: e.workflow,
11051
+ namespace: e.namespace,
11052
+ timeoutMs: e.ms
11053
+ }));
11054
+ this.announcedTimeoutsRead = {
11055
+ at: now,
11056
+ list: list2
11057
+ };
11058
+ return list2;
11059
+ }
11060
+ /** Cancel (`execution_timeout`) the in-flight runs of `workflow` in `namespace` created before `deadline`. */
11061
+ async cancelTimedOut(workflow, namespace, deadline) {
11062
+ const inflight = await this.store.listRuns({
11063
+ workflow,
11064
+ statuses: TIMEOUT_SWEPT_STATUSES,
11065
+ namespace
11066
+ });
11067
+ for (const run of inflight) {
11068
+ if (this.draining) return;
11069
+ if (run.createdAt.getTime() > deadline) continue;
11070
+ const error2 = {
11071
+ message: "execution timeout",
11072
+ code: "execution_timeout"
11073
+ };
11074
+ await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
11075
+ status: "cancelled",
11076
+ error: error2,
11077
+ updatedAt: /* @__PURE__ */ new Date()
11078
+ }));
11079
+ this.emit({
11080
+ type: "run.failed",
11081
+ runId: run.id,
11082
+ workflow: run.workflow,
11083
+ namespace: run.namespace,
11084
+ error: error2
11085
+ });
11086
+ await this.cancelChildren(run.id).catch(() => void 0);
11087
+ this.notifyParent(run.id, cancelledCompletion(error2));
10770
11088
  }
10771
11089
  }
10772
11090
  /**
@@ -13478,7 +13796,7 @@ ${run.workflowVersion}`;
13478
13796
  let wfPlan = await this.planRoute(wfToken, registered.requires);
13479
13797
  if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
13480
13798
  if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
13481
- const checkpoints = await this.replayCheckpoints(run.id);
13799
+ const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
13482
13800
  await this.deliverFinishedChildren(run.id, checkpoints);
13483
13801
  await this.deliverFinishedChildWaiters(run.id, checkpoints);
13484
13802
  const handoff = liveHandoff(run.id, checkpoints);
@@ -13564,6 +13882,10 @@ ${run.workflowVersion}`;
13564
13882
  */
13565
13883
  async applyDecision(run, decision) {
13566
13884
  if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
13885
+ const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
13886
+ if (handoff !== void 0 && decision.status === "completed") {
13887
+ return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
13888
+ }
13567
13889
  if (decision.status === "completed") {
13568
13890
  await this.applyCommands(run, decision.commands);
13569
13891
  return this.settleRun(run, {
@@ -13610,6 +13932,63 @@ ${run.workflowVersion}`;
13610
13932
  return suspended;
13611
13933
  }
13612
13934
  /**
13935
+ * A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
13936
+ * run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
13937
+ * engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
13938
+ * turn replayed stays cancelled and starts nothing.
13939
+ */
13940
+ async applyRemoteContinueAsNew(run, commands, input) {
13941
+ await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
13942
+ const fresh = await this.store.getRun(run.id);
13943
+ if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
13944
+ return {
13945
+ runId: run.id,
13946
+ status: fresh.status,
13947
+ output: fresh.output,
13948
+ error: fresh.error
13949
+ };
13950
+ }
13951
+ const nextId = nextContinuationId(run.id);
13952
+ try {
13953
+ await this.start(run.workflow, input, nextId, {
13954
+ namespace: run.namespace,
13955
+ ...inheritTenant(run)
13956
+ });
13957
+ } catch (err) {
13958
+ const message = err instanceof Error ? err.message : String(err);
13959
+ return this.settleRun(run, {
13960
+ kind: "failed",
13961
+ error: {
13962
+ message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
13963
+ code: "continue_as_new_failed"
13964
+ }
13965
+ });
13966
+ }
13967
+ await this.store.updateRun(run.id, {
13968
+ status: "completed",
13969
+ output: void 0,
13970
+ error: void 0,
13971
+ wakeAt: void 0,
13972
+ updatedAt: /* @__PURE__ */ new Date()
13973
+ });
13974
+ this.emit({
13975
+ type: "run.completed",
13976
+ runId: run.id,
13977
+ workflow: run.workflow,
13978
+ namespace: run.namespace
13979
+ }, {
13980
+ lifecycle: false
13981
+ });
13982
+ void this.notifyParent(run.id, {
13983
+ ok: true,
13984
+ value: void 0
13985
+ });
13986
+ return {
13987
+ runId: run.id,
13988
+ status: "completed"
13989
+ };
13990
+ }
13991
+ /**
13613
13992
  * Re-drive a run whose turn was computed from a history OLDER than the store's.
13614
13993
  *
13615
13994
  * A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
@@ -13627,11 +14006,12 @@ ${run.workflowVersion}`;
13627
14006
  * Scheduled off the current stack so it runs after the caller released the run lease.
13628
14007
  */
13629
14008
  async redriveIfTurnSawStaleHistory(runId, commands) {
13630
- const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
14009
+ const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
13631
14010
  if (blocking.length === 0) return;
13632
14011
  for (const cmd of blocking) {
13633
14012
  const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
13634
- if (checkpoint && checkpoint.status !== "pending") {
14013
+ const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
14014
+ if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
13635
14015
  setTimeout(() => {
13636
14016
  void this.resumeImplicit(runId).catch(() => void 0);
13637
14017
  }, 0).unref?.();
@@ -13880,7 +14260,12 @@ ${run.workflowVersion}`;
13880
14260
  // name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
13881
14261
  name: cp.kind === "signal" ? void 0 : cp.name,
13882
14262
  output: cp.status === "completed" ? cp.output : void 0,
13883
- error: cp.status === "failed" ? cp.error : void 0
14263
+ error: cp.status === "failed" ? cp.error : void 0,
14264
+ // How many times a dispatched step ran: a worker applying a `call` policy decides a retry
14265
+ // from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
14266
+ ...cp.kind === "remote" ? {
14267
+ attempts: cp.attempts
14268
+ } : {}
13884
14269
  });
13885
14270
  } else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
13886
14271
  events.push({
@@ -13937,11 +14322,12 @@ ${run.workflowVersion}`;
13937
14322
  } else if (cmd.kind === "call") {
13938
14323
  const existing = await this.store.getCheckpoint(run.id, cmd.seq);
13939
14324
  if (existing) {
13940
- const leaseAt = redriveLost ? await this.redriveLostCall(run, existing) : void 0;
14325
+ const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
14326
+ const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
13941
14327
  if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
13942
14328
  continue;
13943
14329
  }
13944
- const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14330
+ const lease = this.callLease(cmd.policy);
13945
14331
  await this.store.saveCheckpoint(stepCheckpoint({
13946
14332
  runId: run.id,
13947
14333
  seq: cmd.seq,
@@ -14001,35 +14387,7 @@ ${run.workflowVersion}`;
14001
14387
  }));
14002
14388
  wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14003
14389
  } else if (cmd.kind === "waitSignal") {
14004
- const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
14005
- await this.store.saveCheckpoint(instantCheckpoint({
14006
- runId: run.id,
14007
- seq: cmd.seq,
14008
- name: `signal:${cmd.signal}`,
14009
- kind: "signal",
14010
- output: payload
14011
- }));
14012
- setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
14013
- }, "deliverBuffered");
14014
- const buffered = await this.store.takeBufferedSignal(cmd.signal);
14015
- if (buffered) {
14016
- await deliverBuffered(buffered.payload);
14017
- } else {
14018
- await this.store.putSignalWaiter({
14019
- token: cmd.signal,
14020
- runId: run.id,
14021
- seq: cmd.seq
14022
- });
14023
- const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
14024
- if (lateBuffered) {
14025
- await this.store.removeSignalWaiter({
14026
- token: cmd.signal,
14027
- runId: run.id,
14028
- seq: cmd.seq
14029
- });
14030
- await deliverBuffered(lateBuffered.payload);
14031
- }
14032
- }
14390
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
14033
14391
  } else if (cmd.kind === "waitAny") {
14034
14392
  const state = await enterWaitAny({
14035
14393
  store: this.store,
@@ -14061,6 +14419,100 @@ ${run.workflowVersion}`;
14061
14419
  } else {
14062
14420
  this.redeliverFinishedChild(childId, existingChild);
14063
14421
  }
14422
+ } else if (cmd.kind === "sleepUntil") {
14423
+ const existing = await this.store.getCheckpoint(run.id, cmd.seq);
14424
+ const deadline = existing?.wakeAt ?? cmd.until;
14425
+ if (!existing) {
14426
+ await this.store.saveCheckpoint(stepCheckpoint({
14427
+ runId: run.id,
14428
+ seq: cmd.seq,
14429
+ name: `sleep:${cmd.seq}`,
14430
+ kind: "sleep",
14431
+ status: "pending",
14432
+ attempts: 1,
14433
+ wakeAt: deadline,
14434
+ enqueuedAt: at,
14435
+ startedAt: at,
14436
+ finishedAt: at
14437
+ }));
14438
+ }
14439
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14440
+ } else if (cmd.kind === "waitSignalTimeout") {
14441
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14442
+ const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
14443
+ waitSeq: cmd.seq,
14444
+ token: cmd.signal
14445
+ });
14446
+ if (deadline <= this.clock()) {
14447
+ this.resumeSoon(run.id);
14448
+ continue;
14449
+ }
14450
+ await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
14451
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14452
+ } else if (cmd.kind === "waitEvent") {
14453
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14454
+ const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
14455
+ let deadline;
14456
+ if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
14457
+ deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
14458
+ waitSeq: cmd.seq,
14459
+ token
14460
+ });
14461
+ if (deadline <= this.clock()) {
14462
+ this.resumeSoon(run.id);
14463
+ continue;
14464
+ }
14465
+ }
14466
+ await this.store.putSignalWaiter({
14467
+ token,
14468
+ runId: run.id,
14469
+ seq: cmd.seq
14470
+ });
14471
+ const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
14472
+ if (hit) {
14473
+ await this.store.removeSignalWaiter({
14474
+ token,
14475
+ runId: run.id,
14476
+ seq: cmd.seq
14477
+ });
14478
+ await this.store.saveCheckpoint(instantCheckpoint({
14479
+ runId: run.id,
14480
+ seq: cmd.seq,
14481
+ name: `signal:${token}`,
14482
+ kind: "signal",
14483
+ output: hit.payload
14484
+ }));
14485
+ this.resumeSoon(run.id);
14486
+ } else if (deadline != null) {
14487
+ wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
14488
+ }
14489
+ } else if (cmd.kind === "spawnChild") {
14490
+ if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
14491
+ if (!await this.store.getRun(cmd.childId)) {
14492
+ try {
14493
+ await this.start(cmd.workflow, cmd.input, cmd.childId, {
14494
+ namespace: run.namespace,
14495
+ ...inheritTenant(run),
14496
+ ...cmd.priority !== void 0 ? {
14497
+ priority: cmd.priority
14498
+ } : {}
14499
+ });
14500
+ } catch (err) {
14501
+ const message = err instanceof Error ? err.message : String(err);
14502
+ this.notifyParent(cmd.childId, {
14503
+ ok: false,
14504
+ error: `child workflow "${cmd.workflow}" failed to start: ${message}`
14505
+ });
14506
+ }
14507
+ }
14508
+ await this.store.saveCheckpoint(instantCheckpoint({
14509
+ runId: run.id,
14510
+ seq: cmd.seq,
14511
+ name: `spawn:${cmd.childId}`,
14512
+ kind: "local",
14513
+ output: cmd.childId
14514
+ }));
14515
+ } else if (cmd.kind === "continueAsNew") {
14064
14516
  } else {
14065
14517
  throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
14066
14518
  }
@@ -14068,6 +14520,245 @@ ${run.workflowVersion}`;
14068
14520
  return wakeAt;
14069
14521
  }
14070
14522
  /**
14523
+ * Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
14524
+ * the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
14525
+ * (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
14526
+ * re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
14527
+ * bails).
14528
+ */
14529
+ async parkRemoteOnSignal(run, seq, token) {
14530
+ const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
14531
+ await this.store.saveCheckpoint(instantCheckpoint({
14532
+ runId: run.id,
14533
+ seq,
14534
+ name: `signal:${token}`,
14535
+ kind: "signal",
14536
+ output: payload
14537
+ }));
14538
+ this.resumeSoon(run.id);
14539
+ }, "deliverBuffered");
14540
+ const buffered = await this.store.takeBufferedSignal(token);
14541
+ if (buffered) {
14542
+ await deliverBuffered(buffered.payload);
14543
+ } else {
14544
+ await this.store.putSignalWaiter({
14545
+ token,
14546
+ runId: run.id,
14547
+ seq
14548
+ });
14549
+ const lateBuffered = await this.store.takeBufferedSignal(token);
14550
+ if (lateBuffered) {
14551
+ await this.store.removeSignalWaiter({
14552
+ token,
14553
+ runId: run.id,
14554
+ seq
14555
+ });
14556
+ await deliverBuffered(lateBuffered.payload);
14557
+ }
14558
+ }
14559
+ }
14560
+ /** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
14561
+ resumeSoon(runId) {
14562
+ setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
14563
+ }
14564
+ /**
14565
+ * Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
14566
+ * `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
14567
+ * back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
14568
+ * deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
14569
+ */
14570
+ async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
14571
+ const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
14572
+ if (recorded?.wakeAt != null) return recorded.wakeAt;
14573
+ const deadline = this.clock() + timeoutMs;
14574
+ const at = /* @__PURE__ */ new Date();
14575
+ await this.store.saveCheckpoint(stepCheckpoint({
14576
+ runId,
14577
+ seq: deadlineSeq,
14578
+ name: `timeout:${label}`,
14579
+ kind: "sleep",
14580
+ status: "pending",
14581
+ attempts: 1,
14582
+ input: wait,
14583
+ wakeAt: deadline,
14584
+ enqueuedAt: at,
14585
+ startedAt: at,
14586
+ finishedAt: at
14587
+ }));
14588
+ return deadline;
14589
+ }
14590
+ /**
14591
+ * Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
14592
+ * (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
14593
+ * time out — the in-process wait removes its waiter the same way when it finds the deadline past.
14594
+ * A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
14595
+ * worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
14596
+ */
14597
+ async expireRemoteDeadlines(runId, checkpoints) {
14598
+ const now = this.clock();
14599
+ const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
14600
+ let out = checkpoints;
14601
+ for (const cp of checkpoints) {
14602
+ if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
14603
+ if (cp.wakeAt == null || cp.wakeAt > now) continue;
14604
+ const wait = cp.input;
14605
+ if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
14606
+ if (settled.has(wait.waitSeq)) continue;
14607
+ await this.store.removeSignalWaiter({
14608
+ token: wait.token,
14609
+ runId,
14610
+ seq: wait.waitSeq
14611
+ });
14612
+ const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
14613
+ if (landed && landed.status === "completed") {
14614
+ out = [
14615
+ ...out.filter((c) => c.seq !== landed.seq),
14616
+ landed
14617
+ ].sort((a, b) => a.seq - b.seq);
14618
+ }
14619
+ }
14620
+ return out;
14621
+ }
14622
+ /**
14623
+ * Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
14624
+ * see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
14625
+ * candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
14626
+ * {@link eventBufferTtlMs} is set, like the reconcile sweep.
14627
+ */
14628
+ async claimBufferedEvent(run, name, match) {
14629
+ const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
14630
+ const nowMs = this.clock();
14631
+ for (const candidate of candidates) {
14632
+ if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
14633
+ await this.store.removeBufferedEvent(candidate.id);
14634
+ continue;
14635
+ }
14636
+ const { payload, tenantIds } = readBufferedEvent(candidate.payload);
14637
+ if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
14638
+ if (!eventMatches(payload, match)) continue;
14639
+ if (!await this.store.removeBufferedEvent(candidate.id)) continue;
14640
+ return {
14641
+ payload
14642
+ };
14643
+ }
14644
+ return void 0;
14645
+ }
14646
+ /** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
14647
+ * sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
14648
+ callLease(policy) {
14649
+ if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
14650
+ return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
14651
+ }
14652
+ /**
14653
+ * A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
14654
+ * checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
14655
+ * callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
14656
+ * behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
14657
+ * timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
14658
+ * and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
14659
+ *
14660
+ * - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
14661
+ * the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
14662
+ * dispatched again under the same `stepId` with `attempts + 1`.
14663
+ * - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
14664
+ * re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
14665
+ * workflow can catch, as `RemoteStepTimeout` would surface in-process.
14666
+ */
14667
+ async applyCallPolicy(run, cmd, cp) {
14668
+ const policy = cmd.policy;
14669
+ if (cp.kind !== "remote") return null;
14670
+ if (cp.status === "failed") {
14671
+ if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
14672
+ let due = cp.wakeAt;
14673
+ if (due == null) {
14674
+ due = this.clock() + backoffDelay(cp.attempts, policy);
14675
+ if (due > this.clock()) {
14676
+ await this.store.saveCheckpoint({
14677
+ ...cp,
14678
+ wakeAt: due
14679
+ });
14680
+ return due;
14681
+ }
14682
+ }
14683
+ if (this.clock() < due) return due;
14684
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
14685
+ }
14686
+ if (cp.status !== "pending" || !policy.timeoutMs) return null;
14687
+ if (cp.wakeAt == null) {
14688
+ const lease = this.clock() + policy.timeoutMs;
14689
+ await this.store.saveCheckpoint({
14690
+ ...cp,
14691
+ wakeAt: lease
14692
+ });
14693
+ return this.stepLeaseWakeAt(lease);
14694
+ }
14695
+ if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
14696
+ if (cp.attempts < remoteCallMaxAttempts(policy)) {
14697
+ return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
14698
+ }
14699
+ const id = cp.stepId ?? stepId(run.id, cp.seq);
14700
+ const error2 = {
14701
+ message: new RemoteStepTimeout(id, policy.timeoutMs).message,
14702
+ code: "remote_step_timeout"
14703
+ };
14704
+ await this.store.saveCheckpoint({
14705
+ ...cp,
14706
+ status: "failed",
14707
+ error: error2,
14708
+ wakeAt: void 0,
14709
+ finishedAt: /* @__PURE__ */ new Date()
14710
+ });
14711
+ this.emit({
14712
+ type: "step.failed",
14713
+ runId: run.id,
14714
+ seq: cp.seq,
14715
+ name: cp.name,
14716
+ kind: "remote",
14717
+ error: error2
14718
+ });
14719
+ return void 0;
14720
+ }
14721
+ /** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
14722
+ async redispatchCall(run, cmd, cp, attempt, policy, why) {
14723
+ const lease = this.callLease(policy);
14724
+ const at = /* @__PURE__ */ new Date();
14725
+ await this.store.saveCheckpoint({
14726
+ ...cp,
14727
+ status: "pending",
14728
+ output: void 0,
14729
+ error: void 0,
14730
+ attempts: attempt,
14731
+ wakeAt: lease,
14732
+ enqueuedAt: at,
14733
+ startedAt: at,
14734
+ finishedAt: at,
14735
+ events: why === "timeout" ? [
14736
+ ...cp.events ?? [],
14737
+ lostStepEvent(this.clock(), attempt, "redispatched")
14738
+ ] : cp.events
14739
+ });
14740
+ await this.dispatchRemoteTask({
14741
+ runId: run.id,
14742
+ seq: cp.seq,
14743
+ name: cmd.name,
14744
+ stepId: cp.stepId ?? stepId(run.id, cp.seq),
14745
+ group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
14746
+ input: cp.input,
14747
+ attempt
14748
+ });
14749
+ this.emit({
14750
+ type: "step.started",
14751
+ runId: run.id,
14752
+ seq: cp.seq,
14753
+ name: cmd.name,
14754
+ kind: "remote",
14755
+ ...why === "timeout" ? {
14756
+ redispatched: true
14757
+ } : {}
14758
+ });
14759
+ return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
14760
+ }
14761
+ /**
14071
14762
  * Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
14072
14763
  * exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
14073
14764
  * suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
@@ -18469,6 +19160,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
18469
19160
  PARENT_CLOSE_TAG_PREFIX,
18470
19161
  QueueController,
18471
19162
  REMOTE_EVENT,
19163
+ REMOTE_WORKFLOW_FEATURES,
18472
19164
  RUN_HISTORY_FIXTURE_VERSION,
18473
19165
  RUN_VALUE_FACET_LIMIT,
18474
19166
  RUN_VALUE_FACET_SCAN,
@@ -18626,6 +19318,9 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
18626
19318
  readBufferedEvent,
18627
19319
  readRunsByIds,
18628
19320
  readSearchAttributes,
19321
+ remoteCallMaxAttempts,
19322
+ remoteCallPolicyOf,
19323
+ remoteCallRetries,
18629
19324
  requiredCapabilities,
18630
19325
  resolveConcurrency,
18631
19326
  resolveRouting,