@dudousxd/nestjs-durable-core 0.99.0 → 0.101.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -23,6 +23,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
23
23
  var index_exports = {};
24
24
  __export(index_exports, {
25
25
  ACCUMULATOR_SEEN_MAX: () => ACCUMULATOR_SEEN_MAX,
26
+ ANNOUNCED_SCHEDULE_TAG: () => ANNOUNCED_SCHEDULE_TAG,
26
27
  ANY_WAIT_GROUP: () => ANY_WAIT_GROUP,
27
28
  CHILD_EDGE_PREFIXES: () => CHILD_EDGE_PREFIXES,
28
29
  CONTROL_FLOW_SIGNAL: () => CONTROL_FLOW_SIGNAL,
@@ -31,6 +32,7 @@ __export(index_exports, {
31
32
  CodecStateStore: () => CodecStateStore,
32
33
  ConcurrencyLimitError: () => ConcurrencyLimitError,
33
34
  ContinueAsNew: () => ContinueAsNew,
35
+ DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS: () => DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
34
36
  DEFAULT_CONCURRENCY_STATUSES: () => DEFAULT_CONCURRENCY_STATUSES,
35
37
  DEFAULT_PARTITION: () => DEFAULT_PARTITION,
36
38
  DEFAULT_STALLED_AFTER_MS: () => DEFAULT_STALLED_AFTER_MS,
@@ -69,6 +71,7 @@ __export(index_exports, {
69
71
  LIFECYCLE_TRACE_TAG_PREFIX: () => LIFECYCLE_TRACE_TAG_PREFIX,
70
72
  LIFECYCLE_TRIGGERED_TAG_PREFIX: () => LIFECYCLE_TRIGGERED_TAG_PREFIX,
71
73
  LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX: () => LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
74
+ NO_LIVE_WORKER: () => NO_LIVE_WORKER,
72
75
  NestedWorkflowCallError: () => NestedWorkflowCallError,
73
76
  NonDeterminismError: () => NonDeterminismError,
74
77
  PARENT_CLOSE_TAG_PREFIX: () => PARENT_CLOSE_TAG_PREFIX,
@@ -111,6 +114,7 @@ __export(index_exports, {
111
114
  accumulatorRunId: () => accumulatorRunId,
112
115
  aggregateAnnouncements: () => aggregateAnnouncements,
113
116
  aggregatesCheckpoints: () => aggregatesCheckpoints,
117
+ announcedScheduleId: () => announcedScheduleId,
114
118
  announcementsOf: () => announcementsOf,
115
119
  anyCheckpointName: () => anyCheckpointName,
116
120
  applyAttributeQuery: () => applyAttributeQuery,
@@ -768,7 +772,7 @@ __name(bindWorkflowClass, "bindWorkflowClass");
768
772
  function clientOf(ctor) {
769
773
  const client = bindings.get(ctor);
770
774
  if (!client) {
771
- throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow has booted (or, in a store-less tenant worker, use the injected WorkflowService/engine facade instead).`);
775
+ throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow (an operator, or a tenant's DurableModule) has booted.`);
772
776
  }
773
777
  return client;
774
778
  }
@@ -2605,6 +2609,9 @@ function normalizeDescriptor(raw) {
2605
2609
  ...raw.registrations !== void 0 ? {
2606
2610
  registrations: raw.registrations
2607
2611
  } : {},
2612
+ ...raw.schedules !== void 0 ? {
2613
+ schedules: raw.schedules
2614
+ } : {},
2608
2615
  ...raw.partition !== void 0 ? {
2609
2616
  partition: raw.partition
2610
2617
  } : {},
@@ -2679,12 +2686,30 @@ function canonicalizeForHash(d) {
2679
2686
  ...d.registrations !== void 0 ? {
2680
2687
  registrations: canonicalizeRegistrations(d.registrations)
2681
2688
  } : {},
2689
+ // Conditional like `registrations`: a descriptor without schedules keeps its published hash.
2690
+ ...d.schedules !== void 0 ? {
2691
+ schedules: canonicalSchedules(d.schedules)
2692
+ } : {},
2682
2693
  partition: d.partition ?? null,
2683
2694
  namespace: d.namespace ?? null,
2684
2695
  startedAt: d.startedAt
2685
2696
  };
2686
2697
  }
2687
2698
  __name(canonicalizeForHash, "canonicalizeForHash");
2699
+ function canonicalSchedules(a) {
2700
+ const items = [
2701
+ ...a.items ?? []
2702
+ ].sort((x, y) => x.key.localeCompare(y.key));
2703
+ return {
2704
+ tenant: a.tenant,
2705
+ items: items.map((i) => stableJson(i))
2706
+ };
2707
+ }
2708
+ __name(canonicalSchedules, "canonicalSchedules");
2709
+ function stableJson(value) {
2710
+ return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
2711
+ }
2712
+ __name(stableJson, "stableJson");
2688
2713
  function fnv1a64Hex(input) {
2689
2714
  const OFFSET = 0xcbf29ce484222325n;
2690
2715
  const PRIME = 0x100000001b3n;
@@ -2742,6 +2767,14 @@ function describeWorker(opts) {
2742
2767
  ...registrations.length > 0 ? {
2743
2768
  registrations
2744
2769
  } : {},
2770
+ ...opts.schedules !== void 0 ? {
2771
+ schedules: {
2772
+ tenant: opts.schedules.tenant,
2773
+ items: [
2774
+ ...opts.schedules.items
2775
+ ].sort((a, b) => a.key.localeCompare(b.key))
2776
+ }
2777
+ } : {},
2745
2778
  ...opts.partition ? {
2746
2779
  partition: opts.partition
2747
2780
  } : {},
@@ -4160,6 +4193,12 @@ function matchesTenantPredicates(row, query) {
4160
4193
  __name(matchesTenantPredicates, "matchesTenantPredicates");
4161
4194
 
4162
4195
  // src/schedules.ts
4196
+ var ANNOUNCED_SCHEDULE_TAG = "schedule:announced";
4197
+ function announcedScheduleId(namespace, key) {
4198
+ return `app:${namespace}:${key}`;
4199
+ }
4200
+ __name(announcedScheduleId, "announcedScheduleId");
4201
+ var PARKED_FOR_WORKER = "no_live_worker";
4163
4202
  var ScheduleAlreadyExistsError = class extends Error {
4164
4203
  static {
4165
4204
  __name(this, "ScheduleAlreadyExistsError");
@@ -4300,15 +4339,28 @@ var ScheduleClient = class {
4300
4339
  async trigger(id) {
4301
4340
  const record = await this.mustGet(id);
4302
4341
  const runId = `sched:${id}:manual:${this.now()}`;
4303
- return this.host.start(record.workflow, specOf(record).input, runId, startOptions(record));
4342
+ return this.startFor(record, runId);
4343
+ }
4344
+ /**
4345
+ * Start one run of `record`: through the tenant start path for an announced schedule (its run waits
4346
+ * `blocked` for a live worker of its app instead of failing), else a plain `start`.
4347
+ */
4348
+ startFor(record, runId) {
4349
+ const input = specOf(record).input;
4350
+ const opts = startOptions(record);
4351
+ if (isAnnounced(record) && this.host.startParked) {
4352
+ return this.host.startParked(record.workflow, input, runId, opts);
4353
+ }
4354
+ return this.host.start(record.workflow, input, runId, opts);
4304
4355
  }
4305
4356
  /**
4306
- * Fire every due schedule (in this engine's namespace; all namespaces for an operator engine):
4357
+ * Fire every due schedule (in this engine's namespace; all namespaces for an operator engine; only
4358
+ * those carrying `opts.tag` when given):
4307
4359
  * start its window's run and advance it to the next window. Called by the NestJS timer poller on
4308
4360
  * each tick when `persistedSchedules` is on; call it yourself from any other driver. Returns the
4309
4361
  * run ids started (or re-found, for an idempotent re-fire).
4310
4362
  */
4311
- async tick(nowMs = this.now()) {
4363
+ async tick(nowMs = this.now(), opts = {}) {
4312
4364
  const store = this.requireStore();
4313
4365
  const started = [];
4314
4366
  for (let round = 0; round < 50; round++) {
@@ -4316,6 +4368,9 @@ var ScheduleClient = class {
4316
4368
  const due = await store.listSchedules({
4317
4369
  dueBy: nowMs,
4318
4370
  namespace: this.options.namespace,
4371
+ ...opts.tag !== void 0 ? {
4372
+ tag: opts.tag
4373
+ } : {},
4319
4374
  limit: TICK_BATCH
4320
4375
  });
4321
4376
  for (const record of due) {
@@ -4329,15 +4384,19 @@ var ScheduleClient = class {
4329
4384
  }
4330
4385
  /**
4331
4386
  * When {@link tick} next has a schedule to fire, if one is due by `byMs`: the earliest `nextFireAt`
4332
- * of the unpaused schedules (of this engine's namespace) due by then, else `undefined`. One indexed
4387
+ * of the unpaused schedules (of this engine's namespace, carrying `opts.tag` when given) due by
4388
+ * then, else `undefined`. One indexed
4333
4389
  * read (the due-schedule scan `tick` makes, with a later bound and a limit of one). A store that
4334
4390
  * does not persist schedules has none.
4335
4391
  */
4336
- async nextFireAt(byMs) {
4392
+ async nextFireAt(byMs, opts = {}) {
4337
4393
  if (!this.supported) return void 0;
4338
4394
  const [next] = await this.requireStore().listSchedules({
4339
4395
  dueBy: byMs,
4340
4396
  namespace: this.options.namespace,
4397
+ ...opts.tag !== void 0 ? {
4398
+ tag: opts.tag
4399
+ } : {},
4341
4400
  limit: 1
4342
4401
  });
4343
4402
  return next?.nextFireAt ?? void 0;
@@ -4372,10 +4431,10 @@ var ScheduleClient = class {
4372
4431
  let fired = false;
4373
4432
  const skipMissed = spec.catchup === "skip" && !onTime;
4374
4433
  const beforeWindow = latest < window;
4375
- if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state)) {
4434
+ if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state) && !await this.waitingForWorker(record, state)) {
4376
4435
  runId = `sched:${record.id}:${latest}`;
4377
4436
  try {
4378
- await this.host.start(record.workflow, spec.input, runId, startOptions(record));
4437
+ await this.startFor(record, runId);
4379
4438
  fired = true;
4380
4439
  } catch (error2) {
4381
4440
  if (await this.host.getRun(runId)) fired = true;
@@ -4399,6 +4458,182 @@ var ScheduleClient = class {
4399
4458
  }, record.nextFireAt);
4400
4459
  return fired ? runId : void 0;
4401
4460
  }
4461
+ /**
4462
+ * An announced schedule whose previous run is still parked waiting for a live worker of its app
4463
+ * (the app is down): this window is folded into that run instead of parking one more. However long
4464
+ * the app is down, it finds at most one run per schedule waiting when it comes back, the same
4465
+ * bound `catchup: 'latest'` puts on windows missed while no scheduler polled.
4466
+ */
4467
+ async waitingForWorker(record, state) {
4468
+ if (!isAnnounced(record) || !state.lastRunId) return false;
4469
+ const prev = await this.host.getRun(state.lastRunId);
4470
+ return prev?.status === "blocked" && prev.error?.code === PARKED_FOR_WORKER;
4471
+ }
4472
+ /**
4473
+ * Reconcile the schedules adopted from one app (`namespace`) with what its LIVE workers announce
4474
+ * now (`items`, one per key; the caller resolved disagreeing workers). The rule:
4475
+ * - an announced key with no schedule yet is created (id {@link announcedScheduleId}, tagged
4476
+ * {@link ANNOUNCED_SCHEDULE_TAG}, `catchup: 'latest'`); one whose definition changed is
4477
+ * re-applied like `upsert` (its history kept, its next fire recomputed from now). An equal
4478
+ * definition writes nothing, so any number of replicas announcing the same set is a no-op;
4479
+ * - a pause or resume made from the console survives: a re-applied definition keeps the current
4480
+ * paused state, unless the app's own declared `paused` changed;
4481
+ * - an adopted schedule the app no longer announces is marked missing, and paused ("withdrawn")
4482
+ * once it has stayed missing for `graceMs`. It is never deleted: its history stays, and an app
4483
+ * that announces it again (a rollback) gets it back, resumed unless an operator had paused it.
4484
+ *
4485
+ * Call it only for an app that has at least one live worker stating its schedules: with none (the
4486
+ * app is down, or runs an SDK that does not announce schedules), nothing is withdrawn, and its
4487
+ * schedules keep firing into runs that wait for a worker.
4488
+ */
4489
+ async syncAnnounced(namespace, items, opts) {
4490
+ const store = this.requireStore();
4491
+ const now = opts.nowMs ?? this.now();
4492
+ const out = {
4493
+ adopted: [],
4494
+ restored: [],
4495
+ missing: [],
4496
+ withdrawn: [],
4497
+ invalid: []
4498
+ };
4499
+ const adopted = /* @__PURE__ */ new Map();
4500
+ for (let offset = 0; ; offset += TICK_BATCH) {
4501
+ const page = await store.listSchedules({
4502
+ namespace,
4503
+ tag: ANNOUNCED_SCHEDULE_TAG,
4504
+ limit: TICK_BATCH,
4505
+ offset
4506
+ });
4507
+ for (const record of page) if (isAnnounced(record)) adopted.set(record.id, record);
4508
+ if (page.length < TICK_BATCH) break;
4509
+ }
4510
+ for (const item of items) {
4511
+ const id = announcedScheduleId(namespace, item.key);
4512
+ const existing = adopted.get(id) ?? await store.getSchedule(id);
4513
+ adopted.delete(id);
4514
+ const doc = existing ? existing.state.announce : void 0;
4515
+ if (existing && !doc) {
4516
+ out.invalid.push({
4517
+ key: item.key,
4518
+ error: `schedule id "${id}" is taken`
4519
+ });
4520
+ continue;
4521
+ }
4522
+ const fingerprint = stableJson(item);
4523
+ if (existing && doc && doc.fingerprint === fingerprint) {
4524
+ if (doc.missingSince === void 0 && doc.withdrawnAt === void 0) continue;
4525
+ const resume = doc.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal && existing.paused;
4526
+ const spec = specOf(existing);
4527
+ const window = resume ? nextWindowAfter(spec, now) : void 0;
4528
+ await store.updateSchedule(id, {
4529
+ // The withdrawal note goes; the note it replaced comes back.
4530
+ ...doc.withdrawnAt !== void 0 ? {
4531
+ spec: stripUndefined({
4532
+ ...spec,
4533
+ note: doc.noteBeforeWithdrawal
4534
+ })
4535
+ } : {},
4536
+ ...resume ? {
4537
+ paused: false,
4538
+ nextFireAt: window == null ? null : window + jitterFor(id, window, spec)
4539
+ } : {},
4540
+ state: stripUndefined({
4541
+ ...existing.state,
4542
+ ...resume ? {
4543
+ window
4544
+ } : {},
4545
+ announce: {
4546
+ key: doc.key,
4547
+ fingerprint,
4548
+ declaredPaused: doc.declaredPaused
4549
+ }
4550
+ }),
4551
+ updatedAt: new Date(now)
4552
+ });
4553
+ out.restored.push(id);
4554
+ continue;
4555
+ }
4556
+ const declaredPaused = item.paused === true;
4557
+ const paused = existing === null || existing === void 0 ? declaredPaused : declaredPaused !== (doc?.declaredPaused === true) ? declaredPaused : doc?.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal ? declaredPaused : void 0;
4558
+ try {
4559
+ await this.write({
4560
+ id,
4561
+ workflow: item.workflow,
4562
+ input: item.input,
4563
+ cron: item.cron,
4564
+ every: item.everyMs,
4565
+ timezone: item.timezone,
4566
+ jitter: item.jitterMs,
4567
+ overlap: item.overlap,
4568
+ catchup: "latest",
4569
+ namespace,
4570
+ tags: [
4571
+ ANNOUNCED_SCHEDULE_TAG
4572
+ ],
4573
+ paused,
4574
+ // A note (an operator's pause reason) is kept; a withdrawal note is replaced by the one
4575
+ // it had replaced.
4576
+ note: doc?.withdrawnAt !== void 0 ? doc.noteBeforeWithdrawal : existing ? specOf(existing).note : void 0,
4577
+ // Fires belong to the app, never to whichever request context reached the sync.
4578
+ ...existing ? {} : {
4579
+ tenantId: null
4580
+ }
4581
+ }, existing ?? null, {
4582
+ announce: {
4583
+ key: item.key,
4584
+ fingerprint,
4585
+ declaredPaused
4586
+ }
4587
+ });
4588
+ out.adopted.push(id);
4589
+ } catch (error2) {
4590
+ out.invalid.push({
4591
+ key: item.key,
4592
+ error: errorMessage(error2)
4593
+ });
4594
+ }
4595
+ }
4596
+ for (const record of adopted.values()) {
4597
+ const state = record.state;
4598
+ const doc = state.announce;
4599
+ if (doc.withdrawnAt !== void 0) continue;
4600
+ if (doc.missingSince === void 0) {
4601
+ await store.updateSchedule(record.id, {
4602
+ state: {
4603
+ ...state,
4604
+ announce: {
4605
+ ...doc,
4606
+ missingSince: now
4607
+ }
4608
+ },
4609
+ updatedAt: new Date(now)
4610
+ });
4611
+ out.missing.push(record.id);
4612
+ continue;
4613
+ }
4614
+ if (now - doc.missingSince < opts.graceMs) continue;
4615
+ const spec = {
4616
+ ...specOf(record)
4617
+ };
4618
+ spec.note = `withdrawn: app "${namespace}" no longer announces schedule "${doc.key}"`;
4619
+ await store.updateSchedule(record.id, {
4620
+ paused: true,
4621
+ spec,
4622
+ state: {
4623
+ ...state,
4624
+ announce: stripUndefined({
4625
+ ...doc,
4626
+ withdrawnAt: now,
4627
+ pausedBeforeWithdrawal: record.paused,
4628
+ noteBeforeWithdrawal: specOf(record).note
4629
+ })
4630
+ },
4631
+ updatedAt: new Date(now)
4632
+ });
4633
+ out.withdrawn.push(record.id);
4634
+ }
4635
+ return out;
4636
+ }
4402
4637
  /** `overlap: 'skip'` and the previous run of this schedule is still in flight. */
4403
4638
  async overlapping(spec, state) {
4404
4639
  if (spec.overlap !== "skip" || !state.lastRunId) return false;
@@ -4434,7 +4669,7 @@ var ScheduleClient = class {
4434
4669
  tenantId: tenant.id
4435
4670
  };
4436
4671
  }
4437
- async write(opts, existing) {
4672
+ async write(opts, existing, extraState = {}) {
4438
4673
  const store = this.requireStore();
4439
4674
  const spec = toSpec(opts);
4440
4675
  const now = this.now();
@@ -4450,7 +4685,8 @@ var ScheduleClient = class {
4450
4685
  spec,
4451
4686
  state: stripUndefined({
4452
4687
  ...existing?.state ?? {},
4453
- window
4688
+ window,
4689
+ ...extraState
4454
4690
  }),
4455
4691
  ...this.scheduleTenant(opts, existing),
4456
4692
  createdAt: existing?.createdAt ?? new Date(now),
@@ -4496,6 +4732,10 @@ function specOf(record) {
4496
4732
  return record.spec;
4497
4733
  }
4498
4734
  __name(specOf, "specOf");
4735
+ function isAnnounced(record) {
4736
+ return record.state?.announce !== void 0;
4737
+ }
4738
+ __name(isAnnounced, "isAnnounced");
4499
4739
  function latestWindowAtOrBefore(spec, atMs) {
4500
4740
  if (spec.cron != null) return prevCronFireMs(spec.cron, atMs, spec.timezone);
4501
4741
  if (spec.rrule != null) {
@@ -4570,6 +4810,9 @@ function describe(record) {
4570
4810
  lastError: state.lastError,
4571
4811
  tenantId: record.tenantId ?? void 0,
4572
4812
  tenantKind: record.tenantKind ?? void 0,
4813
+ announcedKey: state.announce?.key,
4814
+ unannouncedSince: state.announce?.missingSince != null ? new Date(state.announce.missingSince) : void 0,
4815
+ withdrawnAt: state.announce?.withdrawnAt != null ? new Date(state.announce.withdrawnAt) : void 0,
4573
4816
  createdAt: record.createdAt,
4574
4817
  updatedAt: record.updatedAt
4575
4818
  });
@@ -6955,6 +7198,20 @@ function runNamespace(namespace) {
6955
7198
  }
6956
7199
  __name(runNamespace, "runNamespace");
6957
7200
  var VERSION_UNDECLARED_TAG = "version:undeclared";
7201
+ var NO_LIVE_WORKER = "no_live_worker";
7202
+ var DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS = 10 * 6e4;
7203
+ var TenantStartInvalid = class TenantStartInvalid2 extends Error {
7204
+ static {
7205
+ __name(this, "TenantStartInvalid");
7206
+ }
7207
+ constructor(cause) {
7208
+ super(cause instanceof Error ? cause.message : String(cause), {
7209
+ cause
7210
+ });
7211
+ this.name = "TenantStartInvalid";
7212
+ }
7213
+ };
7214
+ var NO_LIVE_WORKER_MAX_POLL_MS = 6e4;
6958
7215
  var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
6959
7216
  var STEP_FAN_CHUNK = 200;
6960
7217
  var STEP_EVENTS_PER_BROADCAST = 500;
@@ -7127,6 +7384,13 @@ var WorkflowEngine = class _WorkflowEngine {
7127
7384
  maxRecoveryAttempts;
7128
7385
  unregisteredWorkflowGraceMs;
7129
7386
  remoteAdvanceSilenceMs;
7387
+ /**
7388
+ * The re-drive window of a DISPATCHED workflow turn (a remote worker's `advance` sent over the
7389
+ * broker): {@link WorkflowEngineDeps.remoteAdvanceSilenceMs} when set, else `reconcileMs`. Without
7390
+ * it a run suspended on a turn whose worker died (and whose job was lost with it) had no `wakeAt`
7391
+ * and nothing re-drove it, ever. Heartbeats rearm it like the explicit window.
7392
+ */
7393
+ dispatchedTurnSilenceMs;
7130
7394
  reconcileMs;
7131
7395
  /** Parked workflow bodies kept between turns (see {@link WorkflowEngineDeps.stickyRuns}). */
7132
7396
  sticky;
@@ -7149,6 +7413,10 @@ var WorkflowEngine = class _WorkflowEngine {
7149
7413
  remoteRedispatchMax;
7150
7414
  /** How long a `blocked` run waits before the recovery poll re-checks the live fleet (design §7.5). */
7151
7415
  blockedPollMs;
7416
+ /** Verifies the wire tenant of a start-run, and of a worker's schedule announcement. */
7417
+ verifyTenant;
7418
+ /** Wire tenants whose schedule announcement failed verification, warned about once. */
7419
+ rejectedScheduleTenants = /* @__PURE__ */ new Set();
7152
7420
  workerDescriptorCacheMs;
7153
7421
  descriptorCache = /* @__PURE__ */ new Map();
7154
7422
  /** This engine's control-plane handshake descriptor — negotiated against each worker (design §7.3). */
@@ -7347,6 +7615,7 @@ var WorkflowEngine = class _WorkflowEngine {
7347
7615
  this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
7348
7616
  this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
7349
7617
  this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
7618
+ this.dispatchedTurnSilenceMs = this.remoteAdvanceSilenceMs ?? this.reconcileMs;
7350
7619
  this.sticky = new StickyCache(Math.max(0, deps.stickyRuns ?? DEFAULT_STICKY_RUNS), deps.stickyIdleMs ?? DEFAULT_STICKY_IDLE_MS);
7351
7620
  this.routeReplies = deps.routeReplies ?? true;
7352
7621
  const stalledAfter = deps.stalledAfterMs;
@@ -7358,6 +7627,7 @@ var WorkflowEngine = class _WorkflowEngine {
7358
7627
  this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
7359
7628
  this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
7360
7629
  this.blockedPollMs = deps.blockedPollMs ?? 5e3;
7630
+ this.verifyTenant = deps.verifyTenant;
7361
7631
  this.workerDescriptorCacheMs = deps.workerDescriptorCacheMs ?? 2e3;
7362
7632
  this.cpDescriptor = controlPlaneDescriptor({
7363
7633
  instanceId: `cp-${this.instanceId}`,
@@ -7398,6 +7668,9 @@ var WorkflowEngine = class _WorkflowEngine {
7398
7668
  });
7399
7669
  this.schedules = new ScheduleClient(this.store, {
7400
7670
  start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
7671
+ // An announced (tenant) schedule fires through the tenant start path: with no live worker
7672
+ // of its app, the run waits `blocked` instead of the window being lost.
7673
+ startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true), "startParked"),
7401
7674
  getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
7402
7675
  isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
7403
7676
  tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
@@ -7475,13 +7748,12 @@ var WorkflowEngine = class _WorkflowEngine {
7475
7748
  console.warn(`[nestjs-durable] dropped a start-run for "${message.workflow}": invalid or unauthorized tenant token`);
7476
7749
  return;
7477
7750
  }
7478
- await this.start(message.workflow, message.input, message.runId ?? globalThis.crypto.randomUUID(), {
7479
- namespace: tenant,
7480
- tags: message.tags,
7481
- searchAttributes: message.searchAttributes,
7751
+ await this.startForTenant(
7752
+ message,
7753
+ tenant,
7482
7754
  // The customer a tenant process claims for the run (A4): kept only when its verified
7483
7755
  // token grants `tenancy:*` or `tenancy:<id>`, else the run gets no customer tenant.
7484
- ...message.tenantId !== void 0 ? grantsClientTenant(verified, message.tenantId) ? message.tenantKind !== void 0 ? {
7756
+ message.tenantId !== void 0 ? grantsClientTenant(verified, message.tenantId) ? message.tenantKind !== void 0 ? {
7485
7757
  tenantId: message.tenantId,
7486
7758
  tenantKind: message.tenantKind
7487
7759
  } : {
@@ -7489,7 +7761,7 @@ var WorkflowEngine = class _WorkflowEngine {
7489
7761
  } : {
7490
7762
  tenantId: null
7491
7763
  } : {}
7492
- });
7764
+ );
7493
7765
  });
7494
7766
  this.controlPlane?.onControl((msg) => {
7495
7767
  if (msg.from === this.instanceId) return;
@@ -7572,14 +7844,17 @@ var WorkflowEngine = class _WorkflowEngine {
7572
7844
  * The earliest known time (epoch ms) the poll has work, if it is at or before `byMs`: this
7573
7845
  * namespace's next due timer (`listDueTimers` up to `byMs`, limit one: it returns the most overdue
7574
7846
  * first), the next `blocked` run's recovery poll, and with `persistedSchedules` the next persisted
7575
- * schedule to fire (see {@link ScheduleClient.nextFireAt}). `undefined` when nothing is due by
7847
+ * schedule to fire (see {@link ScheduleClient.nextFireAt}; with `announcedSchedules` alone, the next
7848
+ * schedule adopted from a tenant). `undefined` when nothing is due by
7576
7849
  * then. A poller reads it before waiting longer than its tick, so the wait never outlasts a timer.
7577
7850
  */
7578
7851
  async nextDueAt(byMs, opts = {}) {
7579
7852
  const [timers, blocked, schedule] = await Promise.all([
7580
7853
  this.store.listDueTimers(byMs, this.namespace, 1),
7581
7854
  this.dueBlockedRuns(byMs),
7582
- opts.persistedSchedules ? this.schedules.nextFireAt(byMs) : void 0
7855
+ opts.persistedSchedules ? this.schedules.nextFireAt(byMs) : opts.announcedSchedules && this.schedules.supported ? this.schedules.nextFireAt(byMs, {
7856
+ tag: ANNOUNCED_SCHEDULE_TAG
7857
+ }) : void 0
7583
7858
  ]);
7584
7859
  let next;
7585
7860
  for (const at of [
@@ -9496,7 +9771,16 @@ var WorkflowEngine = class _WorkflowEngine {
9496
9771
  }
9497
9772
  }
9498
9773
  }
9499
- async start(workflow, input, runId, opts) {
9774
+ start(workflow, input, runId, opts) {
9775
+ return this.startWith(workflow, input, runId, opts, false);
9776
+ }
9777
+ /**
9778
+ * {@link start}'s body. `parkUnroutable` is the tenant start path ({@link startForTenant}): a
9779
+ * workflow that nothing here can route yet (no registration, no live worker group serving it at
9780
+ * the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
9781
+ * a start whose caller has already returned is never lost.
9782
+ */
9783
+ async startWith(workflow, input, runId, opts, parkUnroutable) {
9500
9784
  const name = workflowName(workflow);
9501
9785
  const pinned = opts?.version;
9502
9786
  const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
@@ -9505,7 +9789,7 @@ var WorkflowEngine = class _WorkflowEngine {
9505
9789
  const ancestor = await this.findRemoteAncestor(runId);
9506
9790
  if (ancestor) registered = this.synthesizeRemoteChild(name, ancestor.version, ancestor);
9507
9791
  }
9508
- if (!registered && pinned === void 0) {
9792
+ if (!registered) {
9509
9793
  const conventionRun = {
9510
9794
  id: runId,
9511
9795
  workflow: name,
@@ -9520,14 +9804,26 @@ var WorkflowEngine = class _WorkflowEngine {
9520
9804
  createdAt: /* @__PURE__ */ new Date(),
9521
9805
  updatedAt: /* @__PURE__ */ new Date()
9522
9806
  };
9523
- registered = await this.resolveRemoteByConvention(conventionRun, {
9807
+ const convention = await this.resolveRemoteByConvention(conventionRun, {
9524
9808
  askFleetForVersion: true
9525
9809
  });
9810
+ registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
9811
+ }
9812
+ if (!registered && parkUnroutable) {
9813
+ return this.parkStartForWorker(name, input, runId, opts);
9526
9814
  }
9527
9815
  if (!registered) {
9528
9816
  throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
9529
9817
  }
9530
- await registered.validateInput?.(input);
9818
+ if (parkUnroutable && registered.validateInput) {
9819
+ try {
9820
+ await registered.validateInput(input);
9821
+ } catch (err) {
9822
+ throw new TenantStartInvalid(err);
9823
+ }
9824
+ } else {
9825
+ await registered.validateInput?.(input);
9826
+ }
9531
9827
  const prior = await this.store.getRun(runId);
9532
9828
  if (prior) return this.existingStartResult(prior);
9533
9829
  const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency, input);
@@ -9609,6 +9905,187 @@ var WorkflowEngine = class _WorkflowEngine {
9609
9905
  };
9610
9906
  }
9611
9907
  /**
9908
+ * Create the run of a tenant start that nothing can route yet, parked `blocked` with code
9909
+ * {@link NO_LIVE_WORKER} (see {@link startWith}). Idempotent by run id like {@link start}, counts
9910
+ * against a concurrency quota like it, and keeps the start's options on the row: its version (the
9911
+ * pin, else the `'1'` placeholder tagged {@link VERSION_UNDECLARED_TAG} that the first live worker's
9912
+ * declared version replaces), priority, tags, search attributes and customer tenant. A delayed
9913
+ * start is checked first at its `startAt`.
9914
+ */
9915
+ async parkStartForWorker(name, input, runId, opts) {
9916
+ const prior = await this.store.getRun(runId);
9917
+ if (prior) return this.existingStartResult(prior);
9918
+ const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
9919
+ if (quota) await assertConcurrency(this.store, name, quota);
9920
+ const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
9921
+ const group = tenantGroup(sanitizeQueueToken(name), namespace);
9922
+ const pinned = opts?.version;
9923
+ const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
9924
+ const firstCheck = this.clock() + this.blockedPollMs;
9925
+ const wakeAt = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > firstCheck ? startAtMs : firstCheck;
9926
+ const error2 = {
9927
+ message: `no live worker serves ${group}${pinned !== void 0 ? ` at version ${pinned}` : ""} yet; the run starts when one does`,
9928
+ code: NO_LIVE_WORKER,
9929
+ retryable: true
9930
+ };
9931
+ const now = /* @__PURE__ */ new Date();
9932
+ const tags = mergeTags(void 0, [
9933
+ ...opts?.tags ?? [],
9934
+ ...quota ? [
9935
+ concurrencyTag(quota.key)
9936
+ ] : [],
9937
+ ...pinned === void 0 ? [
9938
+ VERSION_UNDECLARED_TAG
9939
+ ] : []
9940
+ ])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
9941
+ const run = {
9942
+ id: runId,
9943
+ workflow: name,
9944
+ workflowVersion: pinned ?? "1",
9945
+ status: "blocked",
9946
+ error: error2,
9947
+ wakeAt,
9948
+ namespace,
9949
+ input,
9950
+ tags,
9951
+ searchAttributes: opts?.searchAttributes,
9952
+ priority: opts?.priority,
9953
+ ...this.startTenant(opts),
9954
+ createdAt: now,
9955
+ updatedAt: now
9956
+ };
9957
+ try {
9958
+ await this.store.createRun(run);
9959
+ } catch (err) {
9960
+ const winner = await this.store.getRun(runId).catch(() => null);
9961
+ if (!winner) throw err;
9962
+ return this.existingStartResult(winner);
9963
+ }
9964
+ this.emit({
9965
+ type: "run.blocked",
9966
+ runId,
9967
+ workflow: name,
9968
+ namespace,
9969
+ error: error2
9970
+ });
9971
+ return {
9972
+ runId,
9973
+ status: "blocked",
9974
+ error: error2
9975
+ };
9976
+ }
9977
+ /**
9978
+ * The control plane's half of a tenant start ({@link StartRunMessage}): start the run with every
9979
+ * option the message carries, under the tenant's namespace. The tenant caller has already returned
9980
+ * `pending`, so nothing here may lose the start:
9981
+ * - a workflow no live worker serves yet is parked `blocked` ({@link NO_LIVE_WORKER}) and started
9982
+ * by the blocked-recovery poll once one is live;
9983
+ * - a start the quota or singleton back-pressure rejects, or whose input a registration here
9984
+ * rejects, is recorded as a `failed` run carrying the reason, so the caller's run id resolves
9985
+ * to an answer instead of nothing (and a deterministic failure is never redelivered);
9986
+ * - a message with no `workflow` is logged and dropped: there is nothing to start;
9987
+ * - any other error (a store blip) is rethrown for the transport to redeliver the message, and a
9988
+ * redelivery is a no-op once the run exists (idempotent by run id).
9989
+ */
9990
+ async startForTenant(message, namespace, tenant) {
9991
+ const runId = message.runId ?? globalThis.crypto.randomUUID();
9992
+ if (typeof message.workflow !== "string" || message.workflow === "") {
9993
+ console.error(`[nestjs-durable] dropped a malformed start-run ${runId}: no workflow name`);
9994
+ return {
9995
+ runId,
9996
+ status: "failed",
9997
+ error: {
9998
+ message: "malformed start-run: no workflow"
9999
+ }
10000
+ };
10001
+ }
10002
+ const opts = {
10003
+ namespace,
10004
+ tags: message.tags,
10005
+ searchAttributes: message.searchAttributes,
10006
+ ...tenant,
10007
+ // Each start option only when the message carries it: a message from an older tenant has none
10008
+ // of them, and starts exactly as it did before.
10009
+ ...typeof message.version === "string" ? {
10010
+ version: message.version
10011
+ } : {},
10012
+ ...typeof message.priority === "number" ? {
10013
+ priority: message.priority
10014
+ } : {},
10015
+ ...typeof message.startAt === "number" ? {
10016
+ startAt: message.startAt
10017
+ } : {},
10018
+ ...message.concurrency !== void 0 ? {
10019
+ concurrency: message.concurrency
10020
+ } : {}
10021
+ };
10022
+ try {
10023
+ return await this.startWith(message.workflow, message.input, runId, opts, true);
10024
+ } catch (err) {
10025
+ if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
10026
+ throw err;
10027
+ }
10028
+ return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
10029
+ }
10030
+ }
10031
+ /** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
10032
+ async recordRejectedStart(name, input, runId, opts, err) {
10033
+ const prior = await this.store.getRun(runId);
10034
+ if (prior) return this.existingStartResult(prior);
10035
+ const error2 = {
10036
+ message: err.message,
10037
+ code: err instanceof ConcurrencyLimitError ? "concurrency_limit" : err instanceof SingletonQueueFullError ? "singleton_queue_full" : "invalid_input",
10038
+ retryable: false
10039
+ };
10040
+ const now = /* @__PURE__ */ new Date();
10041
+ const run = {
10042
+ id: runId,
10043
+ workflow: name,
10044
+ workflowVersion: opts.version ?? "1",
10045
+ status: "failed",
10046
+ error: error2,
10047
+ namespace: poolNamespace(opts.namespace) ?? this.namespace,
10048
+ input,
10049
+ // The same derived tags an admitted or parked start carries, so quota-key queries find it.
10050
+ tags: mergeTags(opts.tags, [
10051
+ ...opts.concurrency ? [
10052
+ concurrencyTag(opts.concurrency.key)
10053
+ ] : [],
10054
+ ...opts.version === void 0 ? [
10055
+ VERSION_UNDECLARED_TAG
10056
+ ] : []
10057
+ ])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag)),
10058
+ searchAttributes: opts.searchAttributes,
10059
+ priority: opts.priority,
10060
+ ...this.startTenant(opts),
10061
+ createdAt: now,
10062
+ updatedAt: now
10063
+ };
10064
+ try {
10065
+ await this.store.createRun(run);
10066
+ } catch (createErr) {
10067
+ const winner = await this.store.getRun(runId).catch(() => null);
10068
+ if (!winner) throw createErr;
10069
+ return this.existingStartResult(winner);
10070
+ }
10071
+ console.warn(`[nestjs-durable] tenant start ${runId} of "${name}" rejected: ${err.message}`);
10072
+ this.emit({
10073
+ type: "run.failed",
10074
+ runId,
10075
+ workflow: name,
10076
+ namespace: run.namespace,
10077
+ error: error2
10078
+ }, {
10079
+ lifecycle: false
10080
+ });
10081
+ void this.deliverRunWaits(runId).catch(() => void 0);
10082
+ return {
10083
+ runId,
10084
+ status: "failed",
10085
+ error: error2
10086
+ };
10087
+ }
10088
+ /**
9612
10089
  * The tenant columns a new run gets: an explicit `opts.tenantId` (`null` = none) wins, else the
9613
10090
  * context tenant when tenancy is on, else none (no keys at all, so a run row is unchanged when
9614
10091
  * tenancy is off).
@@ -10547,6 +11024,32 @@ var WorkflowEngine = class _WorkflowEngine {
10547
11024
  console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
10548
11025
  return this.announceDead(run, error2);
10549
11026
  }
11027
+ /**
11028
+ * Due runs whose workflow nothing could resolve on the last try (`runId` → clock ms of the next
11029
+ * try): a tenant run whose worker is down, say. The timer poll ticks every second and the run stays
11030
+ * due, so without this each tick would lease it, fail to resolve it and log it again. The next try
11031
+ * backs off with the time it has been due ({@link noteUnroutable}). In memory on purpose: the run
11032
+ * row is not touched, so its age toward `unregisteredWorkflowGraceMs` is unchanged, and another
11033
+ * instance (or a restart) simply tries it sooner.
11034
+ */
11035
+ unroutableUntil = /* @__PURE__ */ new Map();
11036
+ /** Whether `runId` is inside its {@link unroutableUntil} back-off at `nowMs`. */
11037
+ unroutableBackoff(runId, nowMs) {
11038
+ const until = this.unroutableUntil.get(runId);
11039
+ if (until === void 0) return false;
11040
+ if (nowMs >= until) {
11041
+ this.unroutableUntil.delete(runId);
11042
+ return false;
11043
+ }
11044
+ return true;
11045
+ }
11046
+ /** Back off the next timer-poll try of `run`, left for later as unresolvable (see {@link unroutableUntil}). */
11047
+ noteUnroutable(run, nowMs) {
11048
+ const dueSince = Math.max(new Date(run.updatedAt).getTime(), run.wakeAt ?? 0);
11049
+ const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor((nowMs - dueSince) / 10)));
11050
+ if (this.unroutableUntil.size > 4096) this.unroutableUntil.clear();
11051
+ this.unroutableUntil.set(run.id, nowMs + delay);
11052
+ }
10550
11053
  /** Log a run skipped for an unregistered workflow — still there, still not progressing. */
10551
11054
  logUnregisteredSkip(run) {
10552
11055
  console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
@@ -10567,8 +11070,8 @@ var WorkflowEngine = class _WorkflowEngine {
10567
11070
  * scan, filtered to just this tick's batch, keeps it cheap regardless of store size.
10568
11071
  */
10569
11072
  async resumeDueTimers(nowMs = this.clock()) {
10570
- const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
10571
- const blocked = await this.dueBlockedRuns(nowMs);
11073
+ const due = shuffle((await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize)).filter((run) => !this.unroutableBackoff(run.id, nowMs)));
11074
+ const blocked = await this.routableBlocked(await this.dueBlockedRuns(nowMs), nowMs);
10572
11075
  if (due.length === 0 && blocked.length === 0) return [];
10573
11076
  if (this.queuesRunsOverTransport && !this.draining) {
10574
11077
  const runs = [
@@ -10670,6 +11173,7 @@ var WorkflowEngine = class _WorkflowEngine {
10670
11173
  const run = await this.store.getRun(runId);
10671
11174
  const settled = run ? await this.settleUnregistered(run, nowMs) : null;
10672
11175
  if (settled !== void 0) return settled;
11176
+ if (run) this.noteUnroutable(run, nowMs);
10673
11177
  if (run && this.isStartOnlyHere(run)) return null;
10674
11178
  }
10675
11179
  throw err;
@@ -10967,6 +11471,7 @@ var WorkflowEngine = class _WorkflowEngine {
10967
11471
  if (error2 instanceof WorkflowNotRegistered) {
10968
11472
  const settled = await this.settleUnregistered(run, this.clock()).catch(() => void 0);
10969
11473
  if (settled) return settled;
11474
+ if (settled === void 0) this.noteUnroutable(run, this.clock());
10970
11475
  if (this.isStartOnlyHere(run)) return void 0;
10971
11476
  }
10972
11477
  console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
@@ -10997,6 +11502,7 @@ var WorkflowEngine = class _WorkflowEngine {
10997
11502
  if (error2 instanceof WorkflowNotRegistered) {
10998
11503
  const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
10999
11504
  if (settled) results.push(settled);
11505
+ if (settled === void 0) this.noteUnroutable(run, nowMs);
11000
11506
  if (settled !== void 0 || this.isStartOnlyHere(run)) continue;
11001
11507
  }
11002
11508
  console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
@@ -11977,6 +12483,75 @@ var WorkflowEngine = class _WorkflowEngine {
11977
12483
  *
11978
12484
  * Workflows only, and deliberately: see the "steps are out of scope" note in `handshake/announced`.
11979
12485
  */
12486
+ /**
12487
+ * Adopt the schedules tenant apps announce (their `schedules` module option, carried on their live
12488
+ * workers' descriptors) as persisted schedules, and withdraw the ones an app stopped announcing.
12489
+ * The control plane's timer poll calls it; {@link ScheduleClient.tick} fires them.
12490
+ *
12491
+ * Per app (the namespace its announcement's verified `tenant` maps to, as for a start-run): the
12492
+ * schedules announced by its live workers, where two workers disagree on a key the one that
12493
+ * started last (its deploy is the newer one) wins. An app with no live worker that states its
12494
+ * schedules is left alone, so its schedules keep firing into runs that wait for a worker. See
12495
+ * {@link ScheduleClient.syncAnnounced} for the reconciliation rule and `graceMs` (default 10
12496
+ * minutes). A namespaced engine adopts only its own namespace's. Nothing without a store that
12497
+ * persists schedules or a transport that reads worker descriptors.
12498
+ */
12499
+ async syncAnnouncedSchedules(opts = {}) {
12500
+ const out = /* @__PURE__ */ new Map();
12501
+ if (!this.schedules.supported) return out;
12502
+ const byNamespace = /* @__PURE__ */ new Map();
12503
+ for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
12504
+ const announced = descriptor.schedules;
12505
+ if (!announced || typeof announced.tenant !== "string" || !Array.isArray(announced.items)) {
12506
+ continue;
12507
+ }
12508
+ let verified;
12509
+ try {
12510
+ verified = await verifyWireTenant(announced.tenant, this.verifyTenant);
12511
+ } catch {
12512
+ verified = null;
12513
+ }
12514
+ const tenant = verified?.tenant ?? null;
12515
+ if (tenant === null) {
12516
+ if (!this.rejectedScheduleTenants.has(descriptor.instanceId)) {
12517
+ this.rejectedScheduleTenants.add(descriptor.instanceId);
12518
+ console.warn(`[nestjs-durable] ignored the schedules worker ${descriptor.instanceId} announces: invalid or unauthorized tenant token`);
12519
+ }
12520
+ continue;
12521
+ }
12522
+ const namespace = poolNamespace(tenant) ?? "default";
12523
+ if (this.namespace !== void 0 && namespace !== this.namespace) continue;
12524
+ const list2 = byNamespace.get(namespace) ?? [];
12525
+ list2.push({
12526
+ startedAt: Number(descriptor.startedAt) || 0,
12527
+ items: announced.items
12528
+ });
12529
+ byNamespace.set(namespace, list2);
12530
+ }
12531
+ for (const [namespace, announcers] of byNamespace) {
12532
+ const items = /* @__PURE__ */ new Map();
12533
+ for (const { items: list2 } of announcers.sort((a, b) => b.startedAt - a.startedAt)) {
12534
+ for (const item of list2) {
12535
+ if (item && typeof item.key === "string" && !items.has(item.key)) {
12536
+ items.set(item.key, item);
12537
+ }
12538
+ }
12539
+ }
12540
+ const result = await this.schedules.syncAnnounced(namespace, [
12541
+ ...items.values()
12542
+ ], {
12543
+ graceMs: opts.graceMs ?? DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
12544
+ ...opts.nowMs !== void 0 ? {
12545
+ nowMs: opts.nowMs
12546
+ } : {}
12547
+ });
12548
+ for (const { key, error: error2 } of result.invalid) {
12549
+ console.warn(`[nestjs-durable] app "${namespace}" announces schedule "${key}", which cannot be adopted: ${error2}`);
12550
+ }
12551
+ out.set(namespace, result);
12552
+ }
12553
+ return out;
12554
+ }
11980
12555
  async announcedWorkflows() {
11981
12556
  return (await this.workflowDirectory()).workflows;
11982
12557
  }
@@ -12916,7 +13491,7 @@ ${run.workflowVersion}`;
12916
13491
  if (remote.executor.dispatch) {
12917
13492
  if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
12918
13493
  const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
12919
- const silenceMs = this.remoteAdvanceSilenceMs;
13494
+ const silenceMs = this.dispatchedTurnSilenceMs;
12920
13495
  const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
12921
13496
  await this.store.updateRun(run.id, await this.withRunTrace(run, {
12922
13497
  status: "suspended",
@@ -14186,6 +14761,73 @@ ${run.workflowVersion}`;
14186
14761
  error: error2
14187
14762
  };
14188
14763
  }
14764
+ /**
14765
+ * Of the due `blocked` runs, the ones to re-drive now. A run parked {@link NO_LIVE_WORKER} (a tenant
14766
+ * start nothing could route) is re-driven only once a live worker serves its group, at its pinned
14767
+ * version when it has one; until then it is re-parked with a longer wait each time, up to
14768
+ * {@link NO_LIVE_WORKER_MAX_POLL_MS}, and never reaches `resume` (which would treat it as an
14769
+ * unregistered workflow and dead-letter it after `unregisteredWorkflowGraceMs`). Every other
14770
+ * blocked run is re-driven as before.
14771
+ */
14772
+ async routableBlocked(blocked, nowMs) {
14773
+ const ready = [];
14774
+ for (const run of blocked) {
14775
+ if (run.error?.code !== NO_LIVE_WORKER) {
14776
+ ready.push(run);
14777
+ continue;
14778
+ }
14779
+ try {
14780
+ const routed = await this.routeParkedStart(run, nowMs);
14781
+ if (routed) ready.push(routed);
14782
+ } catch {
14783
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
14784
+ "blocked"
14785
+ ], {
14786
+ wakeAt: nowMs + this.blockedPollMs
14787
+ }).catch(() => void 0);
14788
+ }
14789
+ }
14790
+ return ready;
14791
+ }
14792
+ /**
14793
+ * One {@link NO_LIVE_WORKER} run: the run to re-drive (its version taken from the live worker when
14794
+ * the start did not pin one and the worker declares it), or `undefined` after re-parking it.
14795
+ */
14796
+ async routeParkedStart(run, nowMs) {
14797
+ if (this.isRegisteredHere(run)) return run;
14798
+ const undeclared = run.tags?.includes(VERSION_UNDECLARED_TAG) === true;
14799
+ const route = await this.resolveRemoteByConvention(run, {
14800
+ askFleetForVersion: true
14801
+ });
14802
+ const routable = route !== void 0 && (undeclared || route.versionDeclared === true && route.version === run.workflowVersion);
14803
+ const updateIf = this.store.updateRunIf ?? this.updateRunIfFallback;
14804
+ if (!routable) {
14805
+ const waited = Math.max(0, nowMs - new Date(run.createdAt).getTime());
14806
+ const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor(waited / 10)));
14807
+ await updateIf.call(this.store, run.id, [
14808
+ "blocked"
14809
+ ], {
14810
+ wakeAt: nowMs + delay
14811
+ });
14812
+ return void 0;
14813
+ }
14814
+ if (undeclared && route.versionDeclared === true) {
14815
+ const tags = run.tags?.filter((tag) => tag !== VERSION_UNDECLARED_TAG);
14816
+ const applied = await updateIf.call(this.store, run.id, [
14817
+ "blocked"
14818
+ ], {
14819
+ workflowVersion: route.version,
14820
+ tags
14821
+ });
14822
+ if (!applied) return void 0;
14823
+ return {
14824
+ ...run,
14825
+ workflowVersion: route.version,
14826
+ tags
14827
+ };
14828
+ }
14829
+ return run;
14830
+ }
14189
14831
  /** Runs parked `blocked` whose recovery poll is due (`wakeAt <= nowMs`), namespace-scoped like
14190
14832
  * {@link resumeDueTimers}. Re-driving one re-checks the live fleet: it dispatches if a
14191
14833
  * capable+compatible worker has appeared, else re-parks with a fresh `wakeAt` (design §7.5). */
@@ -15236,7 +15878,7 @@ ${run.workflowVersion}`;
15236
15878
  * cold (or different) instance re-derives the same answer from the run's own `wakeAt`.
15237
15879
  */
15238
15880
  async rearmDecisionDeadline(runId) {
15239
- const silenceMs = this.remoteAdvanceSilenceMs;
15881
+ const silenceMs = this.dispatchedTurnSilenceMs;
15240
15882
  if (silenceMs == null) return;
15241
15883
  const now = this.clock();
15242
15884
  const lookAgainAt = this.decisionDeadlineRearms.get(runId);
@@ -17773,6 +18415,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
17773
18415
  // Annotate the CommonJS export names for ESM import in node:
17774
18416
  0 && (module.exports = {
17775
18417
  ACCUMULATOR_SEEN_MAX,
18418
+ ANNOUNCED_SCHEDULE_TAG,
17776
18419
  ANY_WAIT_GROUP,
17777
18420
  CHILD_EDGE_PREFIXES,
17778
18421
  CONTROL_FLOW_SIGNAL,
@@ -17781,6 +18424,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
17781
18424
  CodecStateStore,
17782
18425
  ConcurrencyLimitError,
17783
18426
  ContinueAsNew,
18427
+ DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
17784
18428
  DEFAULT_CONCURRENCY_STATUSES,
17785
18429
  DEFAULT_PARTITION,
17786
18430
  DEFAULT_STALLED_AFTER_MS,
@@ -17819,6 +18463,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
17819
18463
  LIFECYCLE_TRACE_TAG_PREFIX,
17820
18464
  LIFECYCLE_TRIGGERED_TAG_PREFIX,
17821
18465
  LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
18466
+ NO_LIVE_WORKER,
17822
18467
  NestedWorkflowCallError,
17823
18468
  NonDeterminismError,
17824
18469
  PARENT_CLOSE_TAG_PREFIX,
@@ -17861,6 +18506,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
17861
18506
  accumulatorRunId,
17862
18507
  aggregateAnnouncements,
17863
18508
  aggregatesCheckpoints,
18509
+ announcedScheduleId,
17864
18510
  announcementsOf,
17865
18511
  anyCheckpointName,
17866
18512
  applyAttributeQuery,