@dudousxd/nestjs-durable-core 0.85.12 → 0.86.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -67,6 +67,7 @@ __export(index_exports, {
67
67
  RunGateway: () => RunGateway,
68
68
  SINGLETON_ACTIVE_STATUSES: () => SINGLETON_ACTIVE_STATUSES,
69
69
  SINGLETON_ADMITTED_TAG: () => SINGLETON_ADMITTED_TAG,
70
+ SINGLETON_LOCK_CHUNK: () => SINGLETON_LOCK_CHUNK,
70
71
  SINGLETON_RESERVED_TAGS: () => SINGLETON_RESERVED_TAGS,
71
72
  SINGLETON_TRACKED_TAG: () => SINGLETON_TRACKED_TAG,
72
73
  STATE_STORE: () => STATE_STORE,
@@ -85,6 +86,7 @@ __export(index_exports, {
85
86
  WORKFLOW_NAME_KEY: () => WORKFLOW_NAME_KEY,
86
87
  WorkflowBlocked: () => WorkflowBlocked,
87
88
  WorkflowEngine: () => WorkflowEngine,
89
+ WorkflowNotRegistered: () => WorkflowNotRegistered,
88
90
  WorkflowSuspended: () => WorkflowSuspended,
89
91
  aggregateAnnouncements: () => aggregateAnnouncements,
90
92
  aggregatesCheckpoints: () => aggregatesCheckpoints,
@@ -199,6 +201,7 @@ __export(index_exports, {
199
201
  signTenantToken: () => signTenantToken,
200
202
  singletonAdmissionDenialPatch: () => singletonAdmissionDenialPatch,
201
203
  singletonAdmissionTags: () => singletonAdmissionTags,
204
+ singletonClaimantIds: () => singletonClaimantIds,
202
205
  sqlComparator: () => sqlComparator,
203
206
  stepConfigOf: () => stepConfigOf,
204
207
  stepId: () => stepId,
@@ -3485,6 +3488,11 @@ function legacySingletonHolders(active, runId, limit) {
3485
3488
  return scanned.filter((run) => run.id !== runId && !run.tags?.includes(SINGLETON_ADMITTED_TAG) && (!run.tags?.includes(SINGLETON_TRACKED_TAG) || window.has(run.id)));
3486
3489
  }
3487
3490
  __name(legacySingletonHolders, "legacySingletonHolders");
3491
+ var SINGLETON_LOCK_CHUNK = 500;
3492
+ function singletonClaimantIds(rows, tag) {
3493
+ return rows.filter((row) => Array.isArray(row.tags) && row.tags.includes(tag)).map((row) => row.id);
3494
+ }
3495
+ __name(singletonClaimantIds, "singletonClaimantIds");
3488
3496
  function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
3489
3497
  if (!Number.isInteger(limit) || limit < 1) throw new Error("singleton limit must be a positive integer");
3490
3498
  const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && run.tags?.includes(tag));
@@ -3529,6 +3537,41 @@ __name(singletonAdmissionDenialPatch, "singletonAdmissionDenialPatch");
3529
3537
  // src/singleton-gate.ts
3530
3538
  var SINGLETON_RETRY_MS = 1e3;
3531
3539
  var SINGLETON_RETRY_JITTER_MS = 250;
3540
+ var ADMISSION_ATTEMPTS = 3;
3541
+ var ADMISSION_BACKOFF_MS = 40;
3542
+ var LOCK_CONFLICT_CODES = /* @__PURE__ */ new Set([
3543
+ "40P01",
3544
+ "40001",
3545
+ "ER_LOCK_DEADLOCK",
3546
+ "1213",
3547
+ "P2034",
3548
+ "TransactionWriteConflict"
3549
+ ]);
3550
+ function isLockConflict(error2) {
3551
+ const seen = /* @__PURE__ */ new Set();
3552
+ const pending = [
3553
+ error2
3554
+ ];
3555
+ while (pending.length > 0) {
3556
+ const current = pending.pop();
3557
+ if (typeof current !== "object" || current === null || seen.has(current)) continue;
3558
+ seen.add(current);
3559
+ const e = current;
3560
+ for (const code of [
3561
+ e.code,
3562
+ e.errno,
3563
+ e.sqlState,
3564
+ e.sqlstate,
3565
+ e.originalCode,
3566
+ e.kind
3567
+ ]) {
3568
+ if (LOCK_CONFLICT_CODES.has(String(code))) return true;
3569
+ }
3570
+ pending.push(e.cause, e.driverError, e.originalError, e.meta, e.driverAdapterError);
3571
+ }
3572
+ return false;
3573
+ }
3574
+ __name(isLockConflict, "isLockConflict");
3532
3575
  var SingletonGate = class {
3533
3576
  static {
3534
3577
  __name(this, "SingletonGate");
@@ -3570,7 +3613,7 @@ var SingletonGate = class {
3570
3613
  if (!this.deps.store.tryAdmitSingleton) {
3571
3614
  throw new Error("Singleton workflows require StateStore.tryAdmitSingleton for atomic durable admission");
3572
3615
  }
3573
- const admitted = await this.deps.store.tryAdmitSingleton(run.id, this.tag(cfg, run.input), run.workflow, cfg.limit ?? 1, this.retryWakeAt());
3616
+ const admitted = await this.claim(run, cfg);
3574
3617
  if (admitted && !run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
3575
3618
  run.tags = [
3576
3619
  ...(run.tags ?? []).filter((tag) => tag !== SINGLETON_TRACKED_TAG),
@@ -3580,6 +3623,26 @@ var SingletonGate = class {
3580
3623
  return admitted;
3581
3624
  }
3582
3625
  /**
3626
+ * One claim, retried when the database aborts it as a lock conflict. A conflict that outlasts the
3627
+ * retries answers "not admitted", so the engine releases this turn's lease, instead of stranding
3628
+ * the run until the lease lapses. It writes nothing to the run: this turn's lease may have lapsed
3629
+ * during the claim, and another instance may now hold the run, its slot and newer tags. A later
3630
+ * sweep re-drives the run: recovery once it is unleased, or its timer if it is parked.
3631
+ */
3632
+ async claim(run, cfg) {
3633
+ const admit = this.deps.store.tryAdmitSingleton;
3634
+ for (let attempt = 1; ; attempt += 1) {
3635
+ try {
3636
+ return await admit.call(this.deps.store, run.id, this.tag(cfg, run.input), run.workflow, cfg.limit ?? 1, this.retryWakeAt());
3637
+ } catch (error2) {
3638
+ if (!isLockConflict(error2)) throw error2;
3639
+ if (attempt >= ADMISSION_ATTEMPTS) return false;
3640
+ const backoff = ADMISSION_BACKOFF_MS * attempt * (0.5 + Math.random());
3641
+ await new Promise((resolve) => setTimeout(resolve, backoff));
3642
+ }
3643
+ }
3644
+ }
3645
+ /**
3583
3646
  * Notify-on-release: a singleton run settled (freeing a slot), so dispatch the oldest gated
3584
3647
  * (`suspended` + same tag) waiters now instead of waiting for their ~1s retry timer. Each re-checks
3585
3648
  * admission in the executor and runs only if it actually wins a slot, so FIFO/race-free guarantees
@@ -5166,6 +5229,22 @@ var NamespaceMismatch = class NamespaceMismatch2 extends Error {
5166
5229
  super("namespace-mismatch");
5167
5230
  }
5168
5231
  };
5232
+ var WorkflowNotRegistered = class extends Error {
5233
+ static {
5234
+ __name(this, "WorkflowNotRegistered");
5235
+ }
5236
+ constructor(workflow, version) {
5237
+ super(`workflow ${workflow}@${version} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
5238
+ this.name = "WorkflowNotRegistered";
5239
+ }
5240
+ };
5241
+ var UNREGISTERED_DEAD_LETTER_FROM = [
5242
+ "pending",
5243
+ "running",
5244
+ "suspended",
5245
+ "cancelling",
5246
+ "blocked"
5247
+ ];
5169
5248
  var WorkflowEngine = class {
5170
5249
  static {
5171
5250
  __name(this, "WorkflowEngine");
@@ -5185,6 +5264,7 @@ var WorkflowEngine = class {
5185
5264
  */
5186
5265
  ownLeaseCutoffMs;
5187
5266
  maxRecoveryAttempts;
5267
+ unregisteredWorkflowGraceMs;
5188
5268
  remoteAdvanceSilenceMs;
5189
5269
  reconcileMs;
5190
5270
  timerBatchSize;
@@ -5371,6 +5451,8 @@ var WorkflowEngine = class {
5371
5451
  this.leaseMs = deps.leaseMs ?? 3e4;
5372
5452
  this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
5373
5453
  this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
5454
+ const grace = deps.unregisteredWorkflowGraceMs;
5455
+ this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
5374
5456
  this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
5375
5457
  this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
5376
5458
  const stalledAfter = deps.stalledAfterMs;
@@ -5687,13 +5769,19 @@ var WorkflowEngine = class {
5687
5769
  * that is lost brings it back due after that period. False when the transport refused the wakes,
5688
5770
  * so the caller resumes them itself.
5689
5771
  */
5690
- async handOffDue(runs, nowMs) {
5772
+ async handOffDue(runs, nowMs, settled) {
5691
5773
  const claimed = [];
5692
5774
  for (const run of runs) {
5693
5775
  if (this.draining) break;
5694
5776
  if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
5695
5777
  continue;
5696
5778
  }
5779
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
5780
+ const result = await this.settleUnregistered(run, nowMs);
5781
+ if (result) settled?.set(run.id, result);
5782
+ else if (result === void 0) this.logUnregisteredSkip(run);
5783
+ continue;
5784
+ }
5697
5785
  try {
5698
5786
  const patch = {
5699
5787
  wakeAt: nowMs + this.leaseMs
@@ -6467,9 +6555,7 @@ var WorkflowEngine = class {
6467
6555
  });
6468
6556
  }
6469
6557
  const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
6470
- if (!registered) {
6471
- throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
6472
- }
6558
+ if (!registered) throw new WorkflowNotRegistered(run.workflow, run.workflowVersion);
6473
6559
  return this.track(this.execute(run, registered.fn));
6474
6560
  }
6475
6561
  /**
@@ -6884,6 +6970,15 @@ var WorkflowEngine = class {
6884
6970
  };
6885
6971
  }
6886
6972
  }
6973
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
6974
+ const settled2 = await this.settleUnregistered(run, nowMs);
6975
+ if (settled2) return {
6976
+ result: settled2,
6977
+ dispatch: false
6978
+ };
6979
+ if (settled2 === void 0) this.logUnregisteredSkip(run);
6980
+ return void 0;
6981
+ }
6887
6982
  const settled = await this.countRecovery(run);
6888
6983
  if (settled) return {
6889
6984
  result: settled,
@@ -6955,25 +7050,10 @@ var WorkflowEngine = class {
6955
7050
  updatedAt: /* @__PURE__ */ new Date()
6956
7051
  });
6957
7052
  await this.store.releaseRunLock(run.id);
6958
- this.emit({
6959
- type: "run.failed",
6960
- runId: run.id,
6961
- workflow: run.workflow,
6962
- namespace: run.namespace,
6963
- error: error2
6964
- });
6965
- this.notifyDead({
7053
+ return this.announceDead({
6966
7054
  ...run,
6967
- status: "dead",
6968
- error: error2,
6969
7055
  recoveryAttempts: attempts
6970
- });
6971
- this.notifyParent(run.id, failedCompletion(error2));
6972
- return {
6973
- runId: run.id,
6974
- status: "dead",
6975
- error: error2
6976
- };
7056
+ }, error2);
6977
7057
  }
6978
7058
  await this.store.updateRun(run.id, {
6979
7059
  recoveryAttempts: attempts,
@@ -6981,6 +7061,85 @@ var WorkflowEngine = class {
6981
7061
  });
6982
7062
  return void 0;
6983
7063
  }
7064
+ /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
7065
+ announceDead(run, error2) {
7066
+ this.emit({
7067
+ type: "run.failed",
7068
+ runId: run.id,
7069
+ workflow: run.workflow,
7070
+ namespace: run.namespace,
7071
+ error: error2
7072
+ });
7073
+ this.notifyDead({
7074
+ ...run,
7075
+ status: "dead",
7076
+ error: error2
7077
+ });
7078
+ this.notifyParent(run.id, failedCompletion(error2));
7079
+ return {
7080
+ runId: run.id,
7081
+ status: "dead",
7082
+ error: error2
7083
+ };
7084
+ }
7085
+ /** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
7086
+ * pays no await (same rule as {@link resume}). */
7087
+ isRegisteredHere(run) {
7088
+ return this.workflows.has(versionKey(run.workflow, run.workflowVersion));
7089
+ }
7090
+ /**
7091
+ * Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
7092
+ * remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
7093
+ */
7094
+ async canResolveWorkflow(run) {
7095
+ if (this.isRegisteredHere(run)) return true;
7096
+ return await this.findInheritedRegistration(run) !== void 0 || await this.resolveRemoteByConvention(run) !== void 0;
7097
+ }
7098
+ /**
7099
+ * A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
7100
+ * {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
7101
+ * rolling deploy's skew: the lease is released at once so the pod that still has the version takes
7102
+ * the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
7103
+ * capable pod would have taken the run long ago): dead-letter it.
7104
+ *
7105
+ * The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
7106
+ * count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
7107
+ * orphan never ages.
7108
+ *
7109
+ * Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
7110
+ * `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
7111
+ */
7112
+ async settleUnregistered(listed, nowMs) {
7113
+ const run = await this.store.getRun(listed.id);
7114
+ if (!run || !UNREGISTERED_DEAD_LETTER_FROM.includes(run.status)) {
7115
+ await this.store.releaseRunLock(listed.id);
7116
+ return null;
7117
+ }
7118
+ const dueSince = Math.max(run.updatedAt.getTime(), run.wakeAt ?? 0);
7119
+ if (this.unregisteredWorkflowGraceMs === 0 || nowMs - dueSince < this.unregisteredWorkflowGraceMs) {
7120
+ await this.store.releaseRunLock(run.id);
7121
+ return void 0;
7122
+ }
7123
+ const error2 = {
7124
+ message: `workflow ${run.workflow}@${run.workflowVersion} is not registered on any live instance (unresumable for over ${this.unregisteredWorkflowGraceMs}ms) \u2014 moved to dead-letter`,
7125
+ code: "workflow_not_registered"
7126
+ };
7127
+ const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
7128
+ run.status
7129
+ ], {
7130
+ status: "dead",
7131
+ error: error2,
7132
+ updatedAt: /* @__PURE__ */ new Date()
7133
+ });
7134
+ await this.store.releaseRunLock(run.id);
7135
+ if (!applied) return null;
7136
+ console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
7137
+ return this.announceDead(run, error2);
7138
+ }
7139
+ /** Log a run skipped for an unregistered workflow — still there, still not progressing. */
7140
+ logUnregisteredSkip(run) {
7141
+ console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
7142
+ }
6984
7143
  /**
6985
7144
  * Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
6986
7145
  * boot. A run still not due re-suspends cheaply without running new work.
@@ -7005,11 +7164,12 @@ var WorkflowEngine = class {
7005
7164
  ...due,
7006
7165
  ...blocked
7007
7166
  ];
7008
- if (await this.handOffDue(runs, nowMs)) {
7009
- return runs.map((run) => ({
7167
+ const settled = /* @__PURE__ */ new Map();
7168
+ if (await this.handOffDue(runs, nowMs, settled)) {
7169
+ return runs.map((run) => settled.get(run.id) ?? {
7010
7170
  runId: run.id,
7011
7171
  status: run.status
7012
- }));
7172
+ });
7013
7173
  }
7014
7174
  }
7015
7175
  const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -7090,8 +7250,13 @@ var WorkflowEngine = class {
7090
7250
  const nowMs = this.clock();
7091
7251
  const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
7092
7252
  if (!acquired) return null;
7093
- return this.resumeImplicit(runId).catch((err) => {
7253
+ return this.resumeImplicit(runId).catch(async (err) => {
7094
7254
  if (err instanceof NamespaceMismatch) return null;
7255
+ if (err instanceof WorkflowNotRegistered) {
7256
+ const run = await this.store.getRun(runId);
7257
+ const settled = run ? await this.settleUnregistered(run, nowMs) : null;
7258
+ if (settled !== void 0) return settled;
7259
+ }
7095
7260
  throw err;
7096
7261
  });
7097
7262
  }
@@ -7346,7 +7511,8 @@ var WorkflowEngine = class {
7346
7511
  * Resume each run only if this instance can acquire its recovery lease — so when several
7347
7512
  * replicas recover or poll at once, each run is picked up by exactly one of them.
7348
7513
  *
7349
- * A run this instance cannot resume is SKIPPED, not allowed to abort the batch. The canonical
7514
+ * A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
7515
+ * stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
7350
7516
  * case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
7351
7517
  * and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
7352
7518
  * and every caller here is a recovery sweep. Letting the first such run end the loop means one
@@ -7370,6 +7536,11 @@ var WorkflowEngine = class {
7370
7536
  const settled = onLocked ? await onLocked(run) : void 0;
7371
7537
  results.push(settled ?? await this.resumeImplicit(run.id));
7372
7538
  } catch (error2) {
7539
+ if (error2 instanceof WorkflowNotRegistered) {
7540
+ const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
7541
+ if (settled) results.push(settled);
7542
+ if (settled !== void 0) continue;
7543
+ }
7373
7544
  console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
7374
7545
  }
7375
7546
  }
@@ -8386,7 +8557,7 @@ var WorkflowEngine = class {
8386
8557
  const current = await this.store.getRun(run.id);
8387
8558
  if (!this.isFenced(run.id)) {
8388
8559
  this.handOffLease(run.id);
8389
- await this.store.releaseRunLock(run.id);
8560
+ await this.store.releaseRunLock(run.id, this.instanceId);
8390
8561
  }
8391
8562
  if (!current) throw new Error(`run ${run.id} was deleted during singleton admission`);
8392
8563
  if (current.status === "suspended") {
@@ -13113,6 +13284,7 @@ var InMemoryRunQueue = class {
13113
13284
  RunGateway,
13114
13285
  SINGLETON_ACTIVE_STATUSES,
13115
13286
  SINGLETON_ADMITTED_TAG,
13287
+ SINGLETON_LOCK_CHUNK,
13116
13288
  SINGLETON_RESERVED_TAGS,
13117
13289
  SINGLETON_TRACKED_TAG,
13118
13290
  STATE_STORE,
@@ -13131,6 +13303,7 @@ var InMemoryRunQueue = class {
13131
13303
  WORKFLOW_NAME_KEY,
13132
13304
  WorkflowBlocked,
13133
13305
  WorkflowEngine,
13306
+ WorkflowNotRegistered,
13134
13307
  WorkflowSuspended,
13135
13308
  aggregateAnnouncements,
13136
13309
  aggregatesCheckpoints,
@@ -13245,6 +13418,7 @@ var InMemoryRunQueue = class {
13245
13418
  signTenantToken,
13246
13419
  singletonAdmissionDenialPatch,
13247
13420
  singletonAdmissionTags,
13421
+ singletonClaimantIds,
13248
13422
  sqlComparator,
13249
13423
  stepConfigOf,
13250
13424
  stepId,