@dudousxd/nestjs-durable-core 0.85.11 → 0.86.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -85,6 +85,7 @@ __export(index_exports, {
85
85
  WORKFLOW_NAME_KEY: () => WORKFLOW_NAME_KEY,
86
86
  WorkflowBlocked: () => WorkflowBlocked,
87
87
  WorkflowEngine: () => WorkflowEngine,
88
+ WorkflowNotRegistered: () => WorkflowNotRegistered,
88
89
  WorkflowSuspended: () => WorkflowSuspended,
89
90
  aggregateAnnouncements: () => aggregateAnnouncements,
90
91
  aggregatesCheckpoints: () => aggregatesCheckpoints,
@@ -5031,6 +5032,8 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
5031
5032
  var VERSION_UNDECLARED_TAG = "version:undeclared";
5032
5033
  var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
5033
5034
  var STEP_FAN_CHUNK = 200;
5035
+ var STEP_EVENTS_PER_BROADCAST = 500;
5036
+ var isStepEvent = /* @__PURE__ */ __name((event) => event.type.startsWith("step."), "isStepEvent");
5034
5037
  var FAN_REREAD_CHUNK = 1e3;
5035
5038
  var STEP_FAN_SEND_ROUND = 2e3;
5036
5039
  var PARENT_NOTIFY_ATTEMPTS = 4;
@@ -5164,6 +5167,22 @@ var NamespaceMismatch = class NamespaceMismatch2 extends Error {
5164
5167
  super("namespace-mismatch");
5165
5168
  }
5166
5169
  };
5170
+ var WorkflowNotRegistered = class extends Error {
5171
+ static {
5172
+ __name(this, "WorkflowNotRegistered");
5173
+ }
5174
+ constructor(workflow, version) {
5175
+ super(`workflow ${workflow}@${version} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
5176
+ this.name = "WorkflowNotRegistered";
5177
+ }
5178
+ };
5179
+ var UNREGISTERED_DEAD_LETTER_FROM = [
5180
+ "pending",
5181
+ "running",
5182
+ "suspended",
5183
+ "cancelling",
5184
+ "blocked"
5185
+ ];
5167
5186
  var WorkflowEngine = class {
5168
5187
  static {
5169
5188
  __name(this, "WorkflowEngine");
@@ -5183,6 +5202,7 @@ var WorkflowEngine = class {
5183
5202
  */
5184
5203
  ownLeaseCutoffMs;
5185
5204
  maxRecoveryAttempts;
5205
+ unregisteredWorkflowGraceMs;
5186
5206
  remoteAdvanceSilenceMs;
5187
5207
  reconcileMs;
5188
5208
  timerBatchSize;
@@ -5369,6 +5389,8 @@ var WorkflowEngine = class {
5369
5389
  this.leaseMs = deps.leaseMs ?? 3e4;
5370
5390
  this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
5371
5391
  this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
5392
+ const grace = deps.unregisteredWorkflowGraceMs;
5393
+ this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
5372
5394
  this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
5373
5395
  this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
5374
5396
  const stalledAfter = deps.stalledAfterMs;
@@ -5501,6 +5523,13 @@ var WorkflowEngine = class {
5501
5523
  ...msg.event,
5502
5524
  at: new Date(msg.event.at)
5503
5525
  }));
5526
+ } else if (msg.kind === "events") {
5527
+ for (const event of msg.events) {
5528
+ this.deliver(markRemoteEvent({
5529
+ ...event,
5530
+ at: new Date(event.at)
5531
+ }));
5532
+ }
5504
5533
  } else if (msg.kind === "cancel") {
5505
5534
  this.notifyCancelled(msg.runId);
5506
5535
  } else if (msg.kind === "enqueued") {
@@ -5678,13 +5707,19 @@ var WorkflowEngine = class {
5678
5707
  * that is lost brings it back due after that period. False when the transport refused the wakes,
5679
5708
  * so the caller resumes them itself.
5680
5709
  */
5681
- async handOffDue(runs, nowMs) {
5710
+ async handOffDue(runs, nowMs, settled) {
5682
5711
  const claimed = [];
5683
5712
  for (const run of runs) {
5684
5713
  if (this.draining) break;
5685
5714
  if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
5686
5715
  continue;
5687
5716
  }
5717
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
5718
+ const result = await this.settleUnregistered(run, nowMs);
5719
+ if (result) settled?.set(run.id, result);
5720
+ else if (result === void 0) this.logUnregisteredSkip(run);
5721
+ continue;
5722
+ }
5688
5723
  try {
5689
5724
  const patch = {
5690
5725
  wakeAt: nowMs + this.leaseMs
@@ -6082,13 +6117,53 @@ var WorkflowEngine = class {
6082
6117
  at: /* @__PURE__ */ new Date()
6083
6118
  };
6084
6119
  this.deliver(full);
6085
- if (this.controlPlane) {
6086
- void this.controlPlane.publishControl({
6087
- kind: "event",
6088
- event: full,
6089
- from: this.instanceId
6090
- }).catch(() => {
6091
- });
6120
+ if (!this.controlPlane) return;
6121
+ this.eventsToBroadcast.push(full);
6122
+ if (!this.broadcastingEvents) void this.broadcastEvents();
6123
+ }
6124
+ /** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
6125
+ eventsToBroadcast = [];
6126
+ broadcastingEvents = false;
6127
+ /**
6128
+ * Publish the queued events in the order they were emitted. A run's own events go out one per
6129
+ * message, as every instance expects them. Step events (a wide fan emits two per step) that queued
6130
+ * up while a publish was in flight go out together in one `events` message, so a fan of 40,000
6131
+ * steps costs the control plane a few hundred messages to every instance instead of 80,000.
6132
+ */
6133
+ async broadcastEvents() {
6134
+ this.broadcastingEvents = true;
6135
+ try {
6136
+ while (this.eventsToBroadcast.length > 0 && this.controlPlane) {
6137
+ const first = this.eventsToBroadcast[0];
6138
+ let msg;
6139
+ if (isStepEvent(first)) {
6140
+ let n = 1;
6141
+ while (n < STEP_EVENTS_PER_BROADCAST && n < this.eventsToBroadcast.length && isStepEvent(this.eventsToBroadcast[n])) {
6142
+ n += 1;
6143
+ }
6144
+ const events = this.eventsToBroadcast.splice(0, n);
6145
+ msg = events.length === 1 ? {
6146
+ kind: "event",
6147
+ event: first,
6148
+ from: this.instanceId
6149
+ } : {
6150
+ kind: "events",
6151
+ events,
6152
+ from: this.instanceId
6153
+ };
6154
+ } else {
6155
+ this.eventsToBroadcast.shift();
6156
+ msg = {
6157
+ kind: "event",
6158
+ event: first,
6159
+ from: this.instanceId
6160
+ };
6161
+ }
6162
+ await this.controlPlane.publishControl(msg).catch(() => {
6163
+ });
6164
+ }
6165
+ } finally {
6166
+ this.broadcastingEvents = false;
6092
6167
  }
6093
6168
  }
6094
6169
  /** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
@@ -6418,9 +6493,7 @@ var WorkflowEngine = class {
6418
6493
  });
6419
6494
  }
6420
6495
  const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
6421
- if (!registered) {
6422
- throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
6423
- }
6496
+ if (!registered) throw new WorkflowNotRegistered(run.workflow, run.workflowVersion);
6424
6497
  return this.track(this.execute(run, registered.fn));
6425
6498
  }
6426
6499
  /**
@@ -6835,6 +6908,15 @@ var WorkflowEngine = class {
6835
6908
  };
6836
6909
  }
6837
6910
  }
6911
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
6912
+ const settled2 = await this.settleUnregistered(run, nowMs);
6913
+ if (settled2) return {
6914
+ result: settled2,
6915
+ dispatch: false
6916
+ };
6917
+ if (settled2 === void 0) this.logUnregisteredSkip(run);
6918
+ return void 0;
6919
+ }
6838
6920
  const settled = await this.countRecovery(run);
6839
6921
  if (settled) return {
6840
6922
  result: settled,
@@ -6906,25 +6988,10 @@ var WorkflowEngine = class {
6906
6988
  updatedAt: /* @__PURE__ */ new Date()
6907
6989
  });
6908
6990
  await this.store.releaseRunLock(run.id);
6909
- this.emit({
6910
- type: "run.failed",
6911
- runId: run.id,
6912
- workflow: run.workflow,
6913
- namespace: run.namespace,
6914
- error: error2
6915
- });
6916
- this.notifyDead({
6991
+ return this.announceDead({
6917
6992
  ...run,
6918
- status: "dead",
6919
- error: error2,
6920
6993
  recoveryAttempts: attempts
6921
- });
6922
- this.notifyParent(run.id, failedCompletion(error2));
6923
- return {
6924
- runId: run.id,
6925
- status: "dead",
6926
- error: error2
6927
- };
6994
+ }, error2);
6928
6995
  }
6929
6996
  await this.store.updateRun(run.id, {
6930
6997
  recoveryAttempts: attempts,
@@ -6932,6 +6999,85 @@ var WorkflowEngine = class {
6932
6999
  });
6933
7000
  return void 0;
6934
7001
  }
7002
+ /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
7003
+ announceDead(run, error2) {
7004
+ this.emit({
7005
+ type: "run.failed",
7006
+ runId: run.id,
7007
+ workflow: run.workflow,
7008
+ namespace: run.namespace,
7009
+ error: error2
7010
+ });
7011
+ this.notifyDead({
7012
+ ...run,
7013
+ status: "dead",
7014
+ error: error2
7015
+ });
7016
+ this.notifyParent(run.id, failedCompletion(error2));
7017
+ return {
7018
+ runId: run.id,
7019
+ status: "dead",
7020
+ error: error2
7021
+ };
7022
+ }
7023
+ /** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
7024
+ * pays no await (same rule as {@link resume}). */
7025
+ isRegisteredHere(run) {
7026
+ return this.workflows.has(versionKey(run.workflow, run.workflowVersion));
7027
+ }
7028
+ /**
7029
+ * Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
7030
+ * remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
7031
+ */
7032
+ async canResolveWorkflow(run) {
7033
+ if (this.isRegisteredHere(run)) return true;
7034
+ return await this.findInheritedRegistration(run) !== void 0 || await this.resolveRemoteByConvention(run) !== void 0;
7035
+ }
7036
+ /**
7037
+ * A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
7038
+ * {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
7039
+ * rolling deploy's skew: the lease is released at once so the pod that still has the version takes
7040
+ * the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
7041
+ * capable pod would have taken the run long ago): dead-letter it.
7042
+ *
7043
+ * The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
7044
+ * count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
7045
+ * orphan never ages.
7046
+ *
7047
+ * Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
7048
+ * `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
7049
+ */
7050
+ async settleUnregistered(listed, nowMs) {
7051
+ const run = await this.store.getRun(listed.id);
7052
+ if (!run || !UNREGISTERED_DEAD_LETTER_FROM.includes(run.status)) {
7053
+ await this.store.releaseRunLock(listed.id);
7054
+ return null;
7055
+ }
7056
+ const dueSince = Math.max(run.updatedAt.getTime(), run.wakeAt ?? 0);
7057
+ if (this.unregisteredWorkflowGraceMs === 0 || nowMs - dueSince < this.unregisteredWorkflowGraceMs) {
7058
+ await this.store.releaseRunLock(run.id);
7059
+ return void 0;
7060
+ }
7061
+ const error2 = {
7062
+ message: `workflow ${run.workflow}@${run.workflowVersion} is not registered on any live instance (unresumable for over ${this.unregisteredWorkflowGraceMs}ms) \u2014 moved to dead-letter`,
7063
+ code: "workflow_not_registered"
7064
+ };
7065
+ const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
7066
+ run.status
7067
+ ], {
7068
+ status: "dead",
7069
+ error: error2,
7070
+ updatedAt: /* @__PURE__ */ new Date()
7071
+ });
7072
+ await this.store.releaseRunLock(run.id);
7073
+ if (!applied) return null;
7074
+ console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
7075
+ return this.announceDead(run, error2);
7076
+ }
7077
+ /** Log a run skipped for an unregistered workflow — still there, still not progressing. */
7078
+ logUnregisteredSkip(run) {
7079
+ console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
7080
+ }
6935
7081
  /**
6936
7082
  * Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
6937
7083
  * boot. A run still not due re-suspends cheaply without running new work.
@@ -6956,11 +7102,12 @@ var WorkflowEngine = class {
6956
7102
  ...due,
6957
7103
  ...blocked
6958
7104
  ];
6959
- if (await this.handOffDue(runs, nowMs)) {
6960
- return runs.map((run) => ({
7105
+ const settled = /* @__PURE__ */ new Map();
7106
+ if (await this.handOffDue(runs, nowMs, settled)) {
7107
+ return runs.map((run) => settled.get(run.id) ?? {
6961
7108
  runId: run.id,
6962
7109
  status: run.status
6963
- }));
7110
+ });
6964
7111
  }
6965
7112
  }
6966
7113
  const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -7041,8 +7188,13 @@ var WorkflowEngine = class {
7041
7188
  const nowMs = this.clock();
7042
7189
  const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
7043
7190
  if (!acquired) return null;
7044
- return this.resumeImplicit(runId).catch((err) => {
7191
+ return this.resumeImplicit(runId).catch(async (err) => {
7045
7192
  if (err instanceof NamespaceMismatch) return null;
7193
+ if (err instanceof WorkflowNotRegistered) {
7194
+ const run = await this.store.getRun(runId);
7195
+ const settled = run ? await this.settleUnregistered(run, nowMs) : null;
7196
+ if (settled !== void 0) return settled;
7197
+ }
7046
7198
  throw err;
7047
7199
  });
7048
7200
  }
@@ -7297,7 +7449,8 @@ var WorkflowEngine = class {
7297
7449
  * Resume each run only if this instance can acquire its recovery lease — so when several
7298
7450
  * replicas recover or poll at once, each run is picked up by exactly one of them.
7299
7451
  *
7300
- * A run this instance cannot resume is SKIPPED, not allowed to abort the batch. The canonical
7452
+ * A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
7453
+ * stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
7301
7454
  * case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
7302
7455
  * and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
7303
7456
  * and every caller here is a recovery sweep. Letting the first such run end the loop means one
@@ -7321,6 +7474,11 @@ var WorkflowEngine = class {
7321
7474
  const settled = onLocked ? await onLocked(run) : void 0;
7322
7475
  results.push(settled ?? await this.resumeImplicit(run.id));
7323
7476
  } catch (error2) {
7477
+ if (error2 instanceof WorkflowNotRegistered) {
7478
+ const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
7479
+ if (settled) results.push(settled);
7480
+ if (settled !== void 0) continue;
7481
+ }
7324
7482
  console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
7325
7483
  }
7326
7484
  }
@@ -13082,6 +13240,7 @@ var InMemoryRunQueue = class {
13082
13240
  WORKFLOW_NAME_KEY,
13083
13241
  WorkflowBlocked,
13084
13242
  WorkflowEngine,
13243
+ WorkflowNotRegistered,
13085
13244
  WorkflowSuspended,
13086
13245
  aggregateAnnouncements,
13087
13246
  aggregatesCheckpoints,